diff --git a/evalscope/evalscope/agent/environments/enclave.py b/evalscope/evalscope/agent/environments/enclave.py index fd2689f..e23ba6e 100644 --- a/evalscope/evalscope/agent/environments/enclave.py +++ b/evalscope/evalscope/agent/environments/enclave.py @@ -19,9 +19,11 @@ shared ``atexit`` hook. from __future__ import annotations +import re import shlex import time -from typing import Any, Dict, List, Optional, Union +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Union from evalscope.api.agent import AgentEnvironment from evalscope.api.agent.types import ExecResult @@ -42,6 +44,64 @@ logger = get_logger() _DEFAULT_DOCKER_IMAGE = 'python:3.11-slim' _DEFAULT_WORKDIR = '/workspace' _DEFAULT_TOOLS: List[str] = ['shell_executor', 'python_executor'] +_DEFAULT_INTERPRETER: List[str] = ['bash', '-c'] +_ENV_KEY_PATTERN = r'[A-Za-z_][A-Za-z0-9_]*' + + +def _unwrap_bash_c(cmd: Any) -> Optional[str]: + """Return the payload for common bash-tool wrappers.""" + if not isinstance(cmd, (list, tuple)) or len(cmd) != 3: + return None + executable, flag, command = cmd + if executable not in {'bash', '/bin/bash'} or flag != '-c' or not isinstance(command, str): + return None + return command + + +def _interpreter_is_bash(interpreter: Sequence[str]) -> bool: + executable = interpreter[0] + return executable == 'bash' or executable.endswith('/bash') + + +def _render_env_exports(env: Dict[str, str]) -> str: + exports = [] + for raw_key, raw_value in env.items(): + key = str(raw_key) + if not re.fullmatch(_ENV_KEY_PATTERN, key): + raise ValueError(f'Invalid environment variable name: {key!r}') + exports.append(f'export {key}={shlex.quote(str(raw_value))};') + return ' '.join(exports) + + +def _render_command( + cmd: List[str], + *, + interpreter: Sequence[str], + cwd: Optional[str], + env: Optional[Dict[str, str]], +) -> str: + unwrapped_command = _unwrap_bash_c(cmd) if _interpreter_is_bash(interpreter) else None + if unwrapped_command is not None: + command = unwrapped_command + elif isinstance(cmd, list): + command = ' '.join(shlex.quote(c) for c in cmd) + else: + command = str(cmd) + if cwd: + command = f'cd {shlex.quote(cwd)} && {command}' + if env: + prefix = _render_env_exports(env) + command = f'{prefix} {command}' if prefix else command + return command + + +def _returncode_from_status(status: Any, stderr: str, timed_out: bool, success_status: Any) -> int: + if status == success_status: + return 0 + if timed_out: + return -1 + match = re.search(r'exit code (\d+)', stderr) + return int(match.group(1)) if match else 1 @register_environment(['enclave', 'docker', 'volcengine']) @@ -63,7 +123,12 @@ class EnclaveAgentEnvironment(AgentEnvironment): (e.g. ``base_url`` / ``region`` / credentials). timeout: Default command timeout (seconds) used when :meth:`exec` is called - without an explicit timeout. + without an explicit timeout. ``None`` uses the environment default. + interpreter: + Command interpreter argv prefix. The rendered command string is + appended as the final argument. Defaults to ``['bash', '-c']`` for + backward compatibility; benchmark adapters may use ``['bash', '-lc']`` + when login-shell initialization is required. """ name: str = 'enclave' @@ -74,7 +139,8 @@ class EnclaveAgentEnvironment(AgentEnvironment): engine: Union[str, SandboxEngine] = SandboxEngine.DOCKER, sandbox_config: Optional[Dict[str, Any]] = None, manager_config: Optional[Dict[str, Any]] = None, - timeout: float = 60.0, + timeout: Optional[float] = None, + interpreter: Optional[Sequence[str]] = None, **_: Any, ) -> None: # ms_enclave is mandatory for this environment. Fail fast at construction @@ -83,7 +149,18 @@ class EnclaveAgentEnvironment(AgentEnvironment): check_import('ms_enclave', extra='sandbox', raise_error=True, feature_name='EnclaveAgentEnvironment') self._engine: SandboxEngine = resolve_engine(engine) - self._timeout = float(timeout) + self._timeout = 60.0 if timeout is None else float(timeout) + if isinstance(interpreter, str): + raise TypeError( + 'EnclaveAgentEnvironment.interpreter must be a non-empty sequence of non-empty strings, ' + 'not a single string.' + ) + self._interpreter: List[str] = list(_DEFAULT_INTERPRETER if interpreter is None else interpreter) + invalid_interpreter = not self._interpreter or any( + not isinstance(part, str) or not part for part in self._interpreter + ) + if invalid_interpreter: + raise ValueError('EnclaveAgentEnvironment.interpreter must be a non-empty sequence of non-empty strings.') self._manager_config: Dict[str, Any] = dict(manager_config or {}) self._sandbox_config_dict: Dict[str, Any] = self._apply_engine_defaults(dict(sandbox_config or {})) self._handle: Optional[SandboxHandle] = None @@ -156,24 +233,12 @@ class EnclaveAgentEnvironment(AgentEnvironment): env: Optional[Dict[str, str]] = None, ) -> ExecResult: handle = await self._ensure_sandbox() - - if isinstance(cmd, list): - command = ' '.join(shlex.quote(c) for c in cmd) - else: - command = str(cmd) - if env: - # Prepend ``KEY=VAL`` pairs so the agent CLI inherits them. - # Using ``env -- ... `` would be cleaner but is not always - # available inside minimal images; inline assignment works in any - # POSIX shell. - prefix = ' '.join(f'{k}={shlex.quote(v)}' for k, v in env.items()) - command = f'{prefix} {command}' if prefix else command - if cwd: - command = f'cd {shlex.quote(cwd)} && {command}' + command = _render_command(cmd, interpreter=self._interpreter, cwd=cwd, env=env) # ms_enclave's shell_executor splits a bare string with no shell - # wrapping; wrap as ``bash -c`` so cd/&&/env-prefix/quoting survive. - shell_argv = ['bash', '-c', command] + # wrapping; use an explicit interpreter so cd/&&/env-prefix/quoting + # survive, while allowing benchmarks to request login-shell setup. + shell_argv = [*self._interpreter, command] timeout_s = float(timeout or self._timeout) @@ -195,15 +260,7 @@ class EnclaveAgentEnvironment(AgentEnvironment): stdout = str(result.output or '') stderr = str(result.error or '') timed_out = result.status == ExecutionStatus.TIMEOUT - - if result.status == ExecutionStatus.SUCCESS: - returncode = 0 - elif timed_out: - returncode = -1 - else: - import re as _re - _m = _re.search(r'exit code (\d+)', stderr) - returncode = int(_m.group(1)) if _m else 1 + returncode = _returncode_from_status(result.status, stderr, timed_out, ExecutionStatus.SUCCESS) # Prefer upstream-reported time when it's a positive number; # otherwise fall back to our locally measured wall-clock. @@ -218,6 +275,13 @@ class EnclaveAgentEnvironment(AgentEnvironment): duration=duration, ) + async def put_dir(self, source_dir: str | Path, target_dir: str) -> None: + """Copy a host directory into the sandbox.""" + handle = await self._ensure_sandbox() + ok = await handle.put_dir(source_dir, target_dir) + if not ok: + raise RuntimeError(f'EnclaveAgentEnvironment.put_dir failed to copy {source_dir} into {target_dir}') + async def close(self) -> None: """Release the per-sample sandbox (idempotent).""" if self._handle is not None: diff --git a/evalscope/evalscope/agent/environments/local.py b/evalscope/evalscope/agent/environments/local.py index 875b4b5..f07b4fc 100644 --- a/evalscope/evalscope/agent/environments/local.py +++ b/evalscope/evalscope/agent/environments/local.py @@ -6,6 +6,9 @@ No container isolation - suitable only for development and CI tests. import asyncio import os +import shutil +import tempfile +from pathlib import Path from typing import Any, Dict, List, Optional from evalscope.api.agent import AgentEnvironment @@ -86,5 +89,38 @@ class LocalAgentEnvironment(AgentEnvironment): async def close(self) -> None: """No external resources to release.""" + async def put_dir(self, source_dir: str | Path, target_dir: str) -> None: + """Copy a host directory into a local target directory.""" + source = Path(source_dir).expanduser() + if not source.is_dir(): + raise FileNotFoundError(f'put_dir source is not a directory: {source}') + target = Path(target_dir).expanduser() + target.mkdir(parents=True, exist_ok=True) + shutil.copytree(source, target, dirs_exist_ok=True) -__all__ = ['LocalAgentEnvironment'] + +class TemporaryLocalAgentEnvironment(LocalAgentEnvironment): + """Local environment backed by a temporary working directory.""" + + def __init__( + self, + sample_id: Any = None, + *, + prefix: str = 'evalscope-local-', + env_vars: Optional[Dict[str, str]] = None, + ) -> None: + raw_sample_id = 'sample' if sample_id is None else str(sample_id) + safe_id = ''.join(char if char.isalnum() else '-' for char in raw_sample_id)[:64] + self._temporary_directory = tempfile.TemporaryDirectory(prefix=f'{prefix}{safe_id}-') + super().__init__(working_dir=self._temporary_directory.name, env_vars=env_vars) + + @property + def working_dir(self) -> Path: + return Path(self._temporary_directory.name) + + async def close(self) -> None: + await super().close() + self._temporary_directory.cleanup() + + +__all__ = ['LocalAgentEnvironment', 'TemporaryLocalAgentEnvironment'] diff --git a/evalscope/evalscope/agent/external/adapter.py b/evalscope/evalscope/agent/external/adapter.py index 2e746ff..d562515 100644 --- a/evalscope/evalscope/agent/external/adapter.py +++ b/evalscope/evalscope/agent/external/adapter.py @@ -15,6 +15,12 @@ import platform import time from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Optional +from evalscope.agent.skills import ( + DEFAULT_SKILLS_INSTALL_DIR, + ResolvedSkills, + format_skills_prompt, + resolve_agent_skills, +) from evalscope.api.agent import AgentEnvironment, AgentTrace from evalscope.api.evaluator import InferenceResult from evalscope.api.messages import ChatMessageAssistant, ChatMessageSystem, ChatMessageUser @@ -47,6 +53,7 @@ def run_external_agent( environment_override: Optional[AgentEnvironment] = None, instruction_override: Optional[str] = None, post_run_hook: Optional[PostRunHook] = None, + close_environment: bool = True, ) -> InferenceResult: """Synchronously drive one sample through an external agent runner. @@ -73,21 +80,40 @@ def run_external_agent( value replaces ``run_result.output`` as the InferenceResult text — the typical use is ``extract_patch(env, cwd)`` for SWE-bench adapters that recover a ``git diff`` from the working tree. + close_environment: + Whether this function owns and closes the environment. Set to + ``False`` when passing a caller-owned ``environment_override`` that + must remain open after the external runner finishes. Uses :class:`AsyncioLoopRunner` to submit the coroutine to the calling thread's long-lived background loop. That loop is reused across samples so the :class:`ModelProxyServer` singleton (which binds to it) only spins up once per worker thread instead of once per sample. """ + if environment_override is None and not close_environment: + raise ValueError('close_environment=False requires environment_override') + instruction = instruction_override if instruction_override is not None else _instruction_from_sample(sample) + skills = resolve_agent_skills( + sample_metadata=sample.metadata, + config_skills_dir=config.skills_dir, + prompt_base_dir=DEFAULT_SKILLS_INSTALL_DIR, + install_paths=[DEFAULT_SKILLS_INSTALL_DIR], + ) + if config.skill_prompt_nudge and skills.enabled: + nudge = format_skills_prompt(skills.skills) + if nudge: + instruction = f'{nudge}\n\n{instruction}' return AsyncioLoopRunner.run( _run_async( config=config, model=model, sample=sample, instruction=instruction, + skills=skills, environment_override=environment_override, post_run_hook=post_run_hook, + close_environment=close_environment, ) ) @@ -97,8 +123,10 @@ async def _run_async( model: Model, sample: 'Sample', instruction: str, + skills: ResolvedSkills, environment_override: Optional[AgentEnvironment], post_run_hook: Optional[PostRunHook], + close_environment: bool, ) -> InferenceResult: runner_cls = get_runner(config.framework) runner_kwargs = dict(config.kwargs) @@ -131,9 +159,13 @@ async def _run_async( task = ExternalAgentTask( instruction=instruction, timeout=config.timeout, - metadata={'sample_id': getattr(sample, 'id', None)}, + metadata={ + 'sample_id': getattr(sample, 'id', None), + 'agent_skills': skills.model_dump(), + }, ) - async with env: + + async def run_in_environment() -> str: session.recorder.record_run_start( framework=config.framework, cmd_summary=runner_cls.__name__, @@ -169,9 +201,14 @@ async def _run_async( # still query the sandbox (e.g. extract a ``git diff``) before # the per-sample environment is closed. if post_run_hook is not None: - final_text = await post_run_hook(env, result, sample) - else: - final_text = result.output + return await post_run_hook(env, result, sample) + return result.output + + if close_environment: + async with env: + final_text = await run_in_environment() + else: + final_text = await run_in_environment() trace: AgentTrace = session.recorder.snapshot() # Prefer the env's own ``name`` (set on the AgentEnvironment subclass) diff --git a/evalscope/evalscope/agent/external/bridge/server.py b/evalscope/evalscope/agent/external/bridge/server.py index 3e4e8b1..1024f45 100644 --- a/evalscope/evalscope/agent/external/bridge/server.py +++ b/evalscope/evalscope/agent/external/bridge/server.py @@ -914,6 +914,10 @@ def _build_generate_config(body: Dict[str, Any]) -> GenerateConfig: kwargs['top_p'] = body['top_p'] if 'stop_sequences' in body and body['stop_sequences']: kwargs['stop_seqs'] = list(body['stop_sequences']) + cache_control = body.get('cache_control') + if isinstance(cache_control, dict): + kwargs['anthropic_cache_control'] = cache_control + kwargs['anthropic_cache_strategy'] = 'recent_messages' return GenerateConfig(**kwargs) diff --git a/evalscope/evalscope/agent/external/bridge/sse_anthropic.py b/evalscope/evalscope/agent/external/bridge/sse_anthropic.py index dc1f95e..50e2e35 100644 --- a/evalscope/evalscope/agent/external/bridge/sse_anthropic.py +++ b/evalscope/evalscope/agent/external/bridge/sse_anthropic.py @@ -25,7 +25,7 @@ from typing import Any, AsyncIterator, Dict, Optional from evalscope.api.model import ModelOutput from ._sse_common import PING_INTERVAL_S, TEXT_CHUNK, TOOL_INPUT_CHUNK, iter_chunks -from .translate_anthropic import map_stop_reason_to_anthropic, unpack_tool_call +from .translate_anthropic import anthropic_usage_payload, map_stop_reason_to_anthropic, unpack_tool_call def _sse(event_type: str, data: Dict[str, Any]) -> bytes: @@ -107,17 +107,13 @@ async def stream_anthropic_response( # 4. message_delta — stop reason + cumulative usage. stop_reason = map_stop_reason_to_anthropic(output.choices[0].stop_reason if output.choices else 'stop') - usage = output.usage delta_payload: Dict[str, Any] = { 'type': 'message_delta', 'delta': { 'stop_reason': stop_reason, 'stop_sequence': None }, - 'usage': { - 'input_tokens': usage.input_tokens if usage else 0, - 'output_tokens': usage.output_tokens if usage else 0, - }, + 'usage': anthropic_usage_payload(output.usage), } yield _sse('message_delta', delta_payload) diff --git a/evalscope/evalscope/agent/external/bridge/translate_anthropic.py b/evalscope/evalscope/agent/external/bridge/translate_anthropic.py index d1cbf49..e9042f3 100644 --- a/evalscope/evalscope/agent/external/bridge/translate_anthropic.py +++ b/evalscope/evalscope/agent/external/bridge/translate_anthropic.py @@ -1,7 +1,8 @@ """Anthropic Messages API ⇄ EvalScope native type translation. -P0 scope: text + ``tool_use`` + ``tool_result`` blocks, no streaming, no -``cache_control``, no extended-thinking blocks. See +P0 scope: text + ``tool_use`` + ``tool_result`` blocks, no extended-thinking +blocks. Anthropic ``cache_control`` markers are preserved through EvalScope +provider-specific ``internal`` / ``options`` fields. See ``.qoder/plans/agent_bridge_design.md`` §7.4 for known-lossy cases. """ @@ -14,6 +15,7 @@ from evalscope.api.messages import ( ChatMessageSystem, ChatMessageTool, ChatMessageUser, + ContentText, ) from evalscope.api.model import ModelOutput from evalscope.api.tool import ToolCall, ToolCallError, ToolFunction, ToolInfo, ToolParams @@ -47,9 +49,10 @@ def anthropic_request_to_messages(body: Dict[str, Any]) -> List[ChatMessage]: if isinstance(system, str) and system: messages.append(ChatMessageSystem(content=system)) elif isinstance(system, list): - text = ''.join(b.get('text', '') for b in system if isinstance(b, dict) and b.get('type') == 'text') - if text: - messages.append(ChatMessageSystem(content=text)) + content = [_content_text_from_block(b) for b in system if isinstance(b, dict) and b.get('type') == 'text'] + content = [c for c in content if c.text] + if content: + messages.append(ChatMessageSystem(content=content)) for entry in body.get('messages') or []: if not isinstance(entry, dict): @@ -68,21 +71,21 @@ def _user_blocks_to_messages(content: Any) -> List[ChatMessage]: return [ChatMessageUser(content=content)] if not isinstance(content, list): return [] - user_text_parts: List[str] = [] + user_content: List[ContentText] = [] tool_msgs: List[ChatMessage] = [] for block in content: if not isinstance(block, dict): continue btype = block.get('type') if btype == 'text': - user_text_parts.append(block.get('text', '')) + user_content.append(_content_text_from_block(block)) elif btype == 'tool_result': tool_msgs.append(_tool_result_to_message(block)) # Tool results precede any new user text so the model sees the # observation first, then the new prompt (matches OpenAI ordering). out: List[ChatMessage] = list(tool_msgs) - if user_text_parts: - out.append(ChatMessageUser(content='\n'.join(p for p in user_text_parts if p))) + if user_content: + out.append(ChatMessageUser(content=_content_or_text(user_content))) return out @@ -98,13 +101,14 @@ def _tool_result_to_message(block: Dict[str, Any]) -> ChatMessageTool: content=text, tool_call_id=block.get('tool_use_id'), error=error, + internal=_anthropic_internal_from_block(block), ) def _assistant_blocks_to_message(content: Any) -> ChatMessageAssistant: if isinstance(content, str): return ChatMessageAssistant(content=content) - text_parts: List[str] = [] + text_content: List[ContentText] = [] tool_calls: List[ToolCall] = [] if isinstance(content, list): for block in content: @@ -112,7 +116,7 @@ def _assistant_blocks_to_message(content: Any) -> ChatMessageAssistant: continue btype = block.get('type') if btype == 'text': - text_parts.append(block.get('text', '')) + text_content.append(_content_text_from_block(block)) elif btype == 'tool_use': tool_calls.append( ToolCall( @@ -121,15 +125,43 @@ def _assistant_blocks_to_message(content: Any) -> ChatMessageAssistant: name=block.get('name', ''), arguments=block.get('input') or {}, ), + internal=_anthropic_internal_from_block(block), type='function', ) ) return ChatMessageAssistant( - content='\n'.join(p for p in text_parts if p), + content=_content_or_text(text_content), tool_calls=tool_calls or None, ) +def _content_text_from_block(block: Dict[str, Any]) -> ContentText: + return ContentText( + text=block.get('text', ''), + internal=_anthropic_internal_from_block(block), + ) + + +def _content_or_text(content: List[ContentText]) -> Any: + if any(_has_anthropic_cache_control(c.internal) for c in content): + return content + return '\n'.join(c.text for c in content if c.text) + + +def _anthropic_internal_from_block(block: Dict[str, Any]) -> Optional[Dict[str, Any]]: + cache_control = block.get('cache_control') + if isinstance(cache_control, dict): + return {'anthropic': {'cache_control': cache_control}} + return None + + +def _has_anthropic_cache_control(internal: Any) -> bool: + return ( + isinstance(internal, dict) and isinstance(internal.get('anthropic'), dict) + and isinstance(internal['anthropic'].get('cache_control'), dict) + ) + + def anthropic_tools_to_tool_infos(tools: Sequence[Dict[str, Any]]) -> List[ToolInfo]: """Translate Anthropic tool specs to ``ToolInfo``. Best-effort: any unparsable parameter schema falls back to an empty ``ToolParams``.""" @@ -142,11 +174,14 @@ def anthropic_tools_to_tool_infos(tools: Sequence[Dict[str, Any]]) -> List[ToolI continue schema = spec.get('input_schema') or {} params = ToolParams() if not isinstance(schema, dict) else _safe_tool_params(schema) - out.append(ToolInfo( - name=name, - description=spec.get('description', '') or '', - parameters=params, - )) + out.append( + ToolInfo( + name=name, + description=spec.get('description', '') or '', + parameters=params, + options=_anthropic_internal_from_block(spec), + ) + ) return out @@ -185,7 +220,7 @@ def model_output_to_anthropic_response( blocks.append({'type': 'text', 'text': ''}) stop_reason = map_stop_reason_to_anthropic(output.choices[0].stop_reason if output.choices else 'stop') - usage = output.usage + usage = anthropic_usage_payload(output.usage) return { 'id': output.id or f'msg_{uuid.uuid4().hex[:24]}', 'type': 'message', @@ -194,10 +229,7 @@ def model_output_to_anthropic_response( 'model': request_model or output.model or '', 'stop_reason': stop_reason, 'stop_sequence': None, - 'usage': { - 'input_tokens': usage.input_tokens if usage else 0, - 'output_tokens': usage.output_tokens if usage else 0, - }, + 'usage': usage, } @@ -218,3 +250,15 @@ def map_stop_reason_to_anthropic(reason: str) -> str: table lives in exactly one place. """ return _STOP_REASON_MAP.get(reason, 'end_turn') + + +def anthropic_usage_payload(usage: Any) -> Dict[str, Any]: + payload: Dict[str, Any] = { + 'input_tokens': usage.input_tokens if usage else 0, + 'output_tokens': usage.output_tokens if usage else 0, + } + if usage and usage.input_tokens_cache_write is not None: + payload['cache_creation_input_tokens'] = usage.input_tokens_cache_write + if usage and usage.input_tokens_cache_read is not None: + payload['cache_read_input_tokens'] = usage.input_tokens_cache_read + return payload diff --git a/evalscope/evalscope/agent/external/runners/claude_code.py b/evalscope/evalscope/agent/external/runners/claude_code.py index 0cb4c7f..fb39def 100644 --- a/evalscope/evalscope/agent/external/runners/claude_code.py +++ b/evalscope/evalscope/agent/external/runners/claude_code.py @@ -19,8 +19,8 @@ from typing import Any, Dict, List, Optional from evalscope.api.agent import AgentEnvironment from evalscope.api.registry import register_runner from evalscope.utils.logger import get_logger -from ._node_install import ensure_node_via_apt from .base import AgentRunner, AgentRunResult, BridgeEndpoint, ExternalAgentTask, RunnerTimeoutError +from .install_helper import ensure_node_via_apt, install_task_skills logger = get_logger() @@ -181,6 +181,13 @@ class ClaudeCodeRunner(AgentRunner): env_vars['HOME'] = home_dir try: + await install_task_skills( + env, + task, + home_dir=home_dir, + native_install_paths=['$HOME/.claude/skills'], + runner_name='ClaudeCodeRunner', + ) # Pass the prompt as the trailing positional argument (matches # claude-code's documented invocation pattern). Avoid variadic # flags like ``--allowedTools `` before the positional diff --git a/evalscope/evalscope/agent/external/runners/codex.py b/evalscope/evalscope/agent/external/runners/codex.py index 3106907..ab827d8 100644 --- a/evalscope/evalscope/agent/external/runners/codex.py +++ b/evalscope/evalscope/agent/external/runners/codex.py @@ -22,8 +22,8 @@ from typing import Any, Dict, List, Optional from evalscope.api.agent import AgentEnvironment from evalscope.api.registry import register_runner from evalscope.utils.logger import get_logger -from ._node_install import ensure_node_via_apt from .base import AgentRunner, AgentRunResult, BridgeEndpoint, ExternalAgentTask, RunnerTimeoutError +from .install_helper import ensure_node_via_apt, install_task_skills logger = get_logger() @@ -170,6 +170,13 @@ class CodexRunner(AgentRunner): home_dir = self._resolve_home() if home_dir is not None: env_vars['HOME'] = home_dir + await install_task_skills( + env, + task, + home_dir=home_dir, + native_install_paths=['$HOME/.agents/skills'], + runner_name='CodexRunner', + ) # Build -c overrides. Order: builtin (provider config) → user extras. # codex parses these as TOML literals, so string values need shell- diff --git a/evalscope/evalscope/agent/external/runners/gemini_cli.py b/evalscope/evalscope/agent/external/runners/gemini_cli.py index 6a23816..db79fb4 100644 --- a/evalscope/evalscope/agent/external/runners/gemini_cli.py +++ b/evalscope/evalscope/agent/external/runners/gemini_cli.py @@ -23,8 +23,8 @@ from typing import Any, Dict, List, Optional from evalscope.api.agent import AgentEnvironment from evalscope.api.registry import register_runner from evalscope.utils.logger import get_logger -from ._node_install import ensure_node_via_apt from .base import AgentRunner, AgentRunResult, BridgeEndpoint, ExternalAgentTask, RunnerTimeoutError +from .install_helper import ensure_node_via_apt, install_task_skills logger = get_logger() @@ -146,6 +146,13 @@ class GeminiCliRunner(AgentRunner): home_dir = self._resolve_home() if home_dir is not None: env_vars['HOME'] = home_dir + await install_task_skills( + env, + task, + home_dir=home_dir, + native_install_paths=['$HOME/.gemini/skills'], + runner_name='GeminiCliRunner', + ) # Build the command. # gemini -p "prompt" executes non-interactively. diff --git a/evalscope/evalscope/agent/external/runners/hermes.py b/evalscope/evalscope/agent/external/runners/hermes.py index 13c33d4..6149eca 100644 --- a/evalscope/evalscope/agent/external/runners/hermes.py +++ b/evalscope/evalscope/agent/external/runners/hermes.py @@ -28,6 +28,7 @@ from evalscope.api.agent import AgentEnvironment from evalscope.api.registry import register_runner from evalscope.utils.logger import get_logger from .base import AgentRunner, AgentRunResult, BridgeEndpoint, ExternalAgentTask, RunnerTimeoutError +from .install_helper import install_task_skills logger = get_logger() @@ -175,6 +176,13 @@ class HermesRunner(AgentRunner): } if home_dir is not None: env_vars['HERMES_HOME'] = home_dir + await install_task_skills( + env, + task, + home_dir=None, + native_install_paths=[], + runner_name='HermesRunner', + ) # Write a config.yaml that points Hermes at the bridge endpoint. # When base_url is set, Hermes ignores the provider and calls the diff --git a/evalscope/evalscope/agent/external/runners/_node_install.py b/evalscope/evalscope/agent/external/runners/install_helper.py similarity index 58% rename from evalscope/evalscope/agent/external/runners/_node_install.py rename to evalscope/evalscope/agent/external/runners/install_helper.py index c86877e..f8427d8 100644 --- a/evalscope/evalscope/agent/external/runners/_node_install.py +++ b/evalscope/evalscope/agent/external/runners/install_helper.py @@ -1,18 +1,13 @@ -"""Shared Node.js probe and nodesource apt installer for Node-based runners. +"""Installation helpers shared by external runners.""" -Four runners (``claude-code``, ``codex``, ``opencode``, ``gemini-cli``) all -need the same two-step sequence when Node.js is absent: install apt -prerequisites (curl, ca-certificates, gnupg) via apt-get, then pull the -nodesource setup script and install ``nodejs``. This module extracts that -common path so the logic lives in one place. -""" - -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional +from evalscope.agent.skills import install_agent_skills, skills_from_sample_metadata from evalscope.utils.logger import get_logger if TYPE_CHECKING: from evalscope.api.agent import AgentEnvironment + from .base import ExternalAgentTask logger = get_logger() @@ -30,18 +25,7 @@ async def ensure_node_via_apt( timeout_s: float, runner_name: str, ) -> None: - """Ensure Node.js and npm are available, installing via nodesource if needed. - - No-ops when ``node`` and ``npm`` are already on PATH. Raises - ``RuntimeError`` when any install step fails (apt prereqs or nodesource). - - Args: - env: The agent execution environment. - node_setup_url: URL of the nodesource distribution setup script - (e.g. ``https://deb.nodesource.com/setup_22.x``). - timeout_s: Wall-clock budget (seconds) for each sub-command. - runner_name: Runner class name used in log and error messages. - """ + """Ensure Node.js and npm are available, installing via nodesource if needed.""" if await node_present(env): return logger.info( @@ -82,4 +66,38 @@ async def ensure_node_via_apt( ) -__all__ = ['ensure_node_via_apt', 'node_present'] +async def install_task_skills( + env: 'AgentEnvironment', + task: 'ExternalAgentTask', + *, + home_dir: Optional[str], + native_install_paths: Optional[List[str]] = None, + runner_name: str, +) -> None: + """Copy task skills into paths visible to the wrapped agent CLI.""" + skills = skills_from_sample_metadata(task.metadata) + if not skills.enabled or not skills.sandbox_dir: + return + + install_paths = _resolve_install_paths( + list(skills.install_paths or []) + list(native_install_paths or []), + home_dir=home_dir, + ) + await install_agent_skills(env, skills, install_paths=install_paths, runner_name=runner_name) + + +def _resolve_install_paths(install_paths: List[str], *, home_dir: Optional[str]) -> List[str]: + resolved: List[str] = [] + seen = set() + for path in install_paths: + if not path: + continue + resolved_path = path.replace('$HOME', home_dir) if home_dir else path + if resolved_path in seen: + continue + seen.add(resolved_path) + resolved.append(resolved_path) + return resolved + + +__all__ = ['ensure_node_via_apt', 'install_task_skills', 'node_present'] diff --git a/evalscope/evalscope/agent/external/runners/mock.py b/evalscope/evalscope/agent/external/runners/mock.py index bcf73ad..c81cb29 100644 --- a/evalscope/evalscope/agent/external/runners/mock.py +++ b/evalscope/evalscope/agent/external/runners/mock.py @@ -14,6 +14,7 @@ from typing import Any, Dict from evalscope.api.registry import register_runner from .base import AgentRunner, AgentRunResult, BridgeEndpoint, ExternalAgentTask, RunnerTimeoutError +from .install_helper import install_task_skills # The body of the mock-agent CLI. Lives as a string so it runs in any # Python interpreter available inside the sandbox (no module import). @@ -97,6 +98,13 @@ class MockAgentRunner(AgentRunner): env_vars['ANTHROPIC_MODEL'] = self._model_name if task.timeout is not None: env_vars['MOCK_TIMEOUT'] = str(task.timeout) + await install_task_skills( + env, + task, + home_dir=None, + native_install_paths=[], + runner_name='MockAgentRunner', + ) result = await env.exec( [sys.executable, '-c', _MOCK_AGENT_SCRIPT], diff --git a/evalscope/evalscope/agent/external/runners/opencode.py b/evalscope/evalscope/agent/external/runners/opencode.py index 0d872d5..1ccded0 100644 --- a/evalscope/evalscope/agent/external/runners/opencode.py +++ b/evalscope/evalscope/agent/external/runners/opencode.py @@ -29,8 +29,8 @@ from typing import Any, Dict, List, Optional from evalscope.api.agent import AgentEnvironment from evalscope.api.registry import register_runner from evalscope.utils.logger import get_logger -from ._node_install import ensure_node_via_apt from .base import AgentRunner, AgentRunResult, BridgeEndpoint, ExternalAgentTask, RunnerTimeoutError +from .install_helper import ensure_node_via_apt, install_task_skills logger = get_logger() @@ -164,6 +164,13 @@ class OpenCodeRunner(AgentRunner): env_vars['HOME'] = home_dir try: + await install_task_skills( + env, + task, + home_dir=home_dir, + native_install_paths=['$HOME/.config/opencode/skills', '$HOME/.opencode/skill'], + runner_name='OpenCodeRunner', + ) # Write opencode.json config to register the model + baseURL. config = { 'provider': { diff --git a/evalscope/evalscope/agent/runner.py b/evalscope/evalscope/agent/runner.py index f9a794a..1657ebc 100644 --- a/evalscope/evalscope/agent/runner.py +++ b/evalscope/evalscope/agent/runner.py @@ -13,11 +13,19 @@ per-benchmark overrides. from typing import TYPE_CHECKING, Any, Callable, Dict, Optional +from evalscope.agent.skills import ( + DEFAULT_SKILLS_INSTALL_DIR, + format_skills_prompt, + install_agent_skills, + resolve_agent_skills, +) +from evalscope.agent.tools.bash import apply_bash_command_timeout_defaults from evalscope.api.agent import AgentEnvironment, AgentLoopResult, run_agent_loop from evalscope.api.evaluator import InferenceResult from evalscope.api.messages import ChatMessageUser from evalscope.api.model import Model from evalscope.api.registry import get_environment, get_strategy, resolve_tool_infos, resolve_tools +from evalscope.utils.function_utils import AsyncioLoopRunner from evalscope.utils.logger import get_logger if TYPE_CHECKING: @@ -34,6 +42,7 @@ def run_native_agent( sample: 'Sample', build_sandbox_config: Callable[['Sample'], Optional[Dict[str, Any]]], extract_final_answer: Callable[[AgentLoopResult, Any], str], + environment_override: Optional[AgentEnvironment] = None, ) -> InferenceResult: """Drive a sample through the native AgentLoop and return its result. @@ -62,27 +71,43 @@ def run_native_agent( # Resolve ToolInfo schemas from the registry so the model can see them. registered_tool_infos = resolve_tool_infos(cfg.tools) - # Determine environment class (if any) – instantiated below so its - # constructor sees the fully merged kwargs. - env_cls: Optional[type] = None - if cfg.environment is not None: + environment: Optional[AgentEnvironment] = environment_override + if environment is None and cfg.environment is not None: env_cls = get_environment(cfg.environment) - - env_kwargs = _resolve_env_kwargs( - task_config=task_config, - sample=sample, - build_sandbox_config=build_sandbox_config, - ) - environment: Optional[AgentEnvironment] = env_cls(**env_kwargs) if env_cls is not None else None + env_kwargs = _resolve_env_kwargs( + task_config=task_config, + sample=sample, + build_sandbox_config=build_sandbox_config, + ) + environment = env_cls(**env_kwargs) + owns_environment = environment_override is None if isinstance(sample.input, list): initial_messages = list(sample.input) else: initial_messages = [ChatMessageUser(content=sample.input)] + skills = resolve_agent_skills( + sample_metadata=sample.metadata, + config_skills_dir=cfg.skills_dir, + prompt_base_dir=DEFAULT_SKILLS_INSTALL_DIR, + install_paths=[DEFAULT_SKILLS_INSTALL_DIR], + ) + if cfg.skill_prompt_nudge and skills.enabled: + nudge = format_skills_prompt(skills.skills) + if nudge: + initial_messages.insert(0, ChatMessageUser(content=nudge)) + if environment is not None: + try: + AsyncioLoopRunner.run(install_agent_skills(environment, skills, runner_name='NativeAgentRunner')) + except Exception: + if owns_environment: + AsyncioLoopRunner.run(environment.close()) + raise # Merge sample-level tools with agent-config tools. sample_tools = list(sample.tools or []) all_tools = sample_tools + [t for t in registered_tool_infos if t not in sample_tools] + handlers, all_tools = apply_bash_command_timeout_defaults(handlers, all_tools, cfg.command_timeout) result: AgentLoopResult = run_agent_loop( model=model, @@ -96,6 +121,7 @@ def run_native_agent( trace_strategy_name=cfg.strategy, trace_env_name=cfg.environment, mcp_configs=list(cfg.mcp_servers) or None, + close_environment=owns_environment, ) final_text = extract_final_answer(result, strategy) @@ -114,10 +140,11 @@ def _resolve_env_kwargs( Precedence (lowest -> highest): 1. ``task_config.sandbox`` — engine / default_config / manager_config - carried alongside the pooled SandboxMixin so sandbox settings are + carried alongside the pooled CodeExecutionSandboxMixin so sandbox settings are defined **once** at the task level. 2. ``build_sandbox_config(sample)`` — per-sample override hook. - 3. ``agent_config.environment_extra`` — raw kwargs forwarded verbatim + 3. ``agent_config.command_timeout`` — default timeout for command-style environments. + 4. ``agent_config.environment_extra`` — raw kwargs forwarded verbatim to the environment constructor (last word for power users). """ env_kwargs: Dict[str, Any] = {} @@ -140,6 +167,9 @@ def _resolve_env_kwargs( if merged_sandbox_cfg: env_kwargs['sandbox_config'] = merged_sandbox_cfg + if task_config.agent_config.command_timeout is not None: + env_kwargs['timeout'] = task_config.agent_config.command_timeout + # environment_extra wins over everything above. env_kwargs.update(task_config.agent_config.environment_extra) return env_kwargs diff --git a/evalscope/evalscope/agent/skills.py b/evalscope/evalscope/agent/skills.py new file mode 100644 index 0000000..5a89f82 --- /dev/null +++ b/evalscope/evalscope/agent/skills.py @@ -0,0 +1,234 @@ +"""Helpers for Agent Skills directory compatibility.""" + +from __future__ import annotations + +import re +import shlex +import yaml +from pathlib import Path +from pydantic import BaseModel, Field +from typing import TYPE_CHECKING, Any, Dict, Iterable, List + +if TYPE_CHECKING: + from evalscope.api.agent import AgentEnvironment + +CONFIG_SKILL_SOURCE = 'config' +TASK_BUNDLED_SKILL_SOURCE = 'task_bundled' +DEFAULT_SKILLS_SANDBOX_DIR = '/tmp/evalscope-agent-skills' +DEFAULT_SKILLS_INSTALL_DIR = '$HOME/.agents/skills' + + +class SkillMetadata(BaseModel): + """Metadata discovered from a ``SKILL.md`` file.""" + + name: str + description: str = '' + path: str + + +class ResolvedSkills(BaseModel): + """Resolved skills for one agent run.""" + + enabled: bool = False + source: str = 'none' + host_dir: str | None = None + sandbox_dir: str | None = None + prompt_base_dir: str | None = None + install_paths: List[str] = Field(default_factory=list) + skills: List[SkillMetadata] = Field(default_factory=list) + metadata_errors: List[str] = Field(default_factory=list) + + +def discover_skills(skills_dir: str | Path, *, path_prefix: str | None = None) -> tuple[List[SkillMetadata], List[str]]: + """Discover immediate child skill directories containing ``SKILL.md``.""" + base = Path(skills_dir) + skills: List[SkillMetadata] = [] + errors: List[str] = [] + if not base.is_dir(): + return skills, [f'skills_dir is not a directory: {base}'] + + for skill_dir in sorted(path for path in base.iterdir() if path.is_dir()): + skill_file = skill_dir / 'SKILL.md' + if not skill_file.is_file(): + continue + try: + content = skill_file.read_text(encoding='utf-8') + except OSError as exc: + errors.append(f'{skill_file}: {exc}') + continue + frontmatter = parse_frontmatter(content) + name = frontmatter.get('name') or skill_dir.name + description = frontmatter.get('description') or '' + if not frontmatter.get('name') or not frontmatter.get('description'): + errors.append(f'{skill_file}: missing name or description frontmatter') + display_base = (path_prefix.rstrip('/') if path_prefix else str(skill_dir.parent)) + skills.append( + SkillMetadata( + name=name, + description=description, + path=f'{display_base}/{skill_dir.name}/SKILL.md', + ) + ) + return skills, errors + + +def parse_frontmatter(content: str) -> Dict[str, str]: + """Parse YAML frontmatter key/value pairs from a skill file.""" + match = re.match(r'^---\s*\n(.*?)\n---', content, re.DOTALL) + if not match: + return {} + parsed = yaml.safe_load(match.group(1)) or {} + if not isinstance(parsed, dict): + return {} + return {str(key): '' if value is None else str(value) for key, value in parsed.items()} + + +def format_skills_prompt(skills: List[SkillMetadata]) -> str: + """Render a neutral prompt nudge for available skills.""" + if not skills: + return '' + lines = [ + 'You have access to the following skills. Each skill is a directory containing a SKILL.md file with ' + 'instructions and resources. When a skill is relevant, read its SKILL.md before using it.', + '', + ] + for skill in skills: + description = f': {skill.description}' if skill.description else '' + lines.append(f'- {skill.name}{description} ({skill.path})') + return '\n'.join(lines) + + +def skills_from_sample_metadata(metadata: Dict[str, Any]) -> ResolvedSkills: + """Build ``ResolvedSkills`` from sample metadata.""" + raw = metadata.get('agent_skills') or {} + if isinstance(raw, ResolvedSkills): + return raw + if isinstance(raw, dict): + try: + return ResolvedSkills.model_validate(raw) + except Exception: + return ResolvedSkills() + return ResolvedSkills() + + +def resolve_agent_skills( + *, + sample_metadata: Dict[str, Any], + config_skills_dir: str | None, + prompt_base_dir: str = DEFAULT_SKILLS_INSTALL_DIR, + install_paths: Iterable[str] = (DEFAULT_SKILLS_INSTALL_DIR, ), + sandbox_dir: str = DEFAULT_SKILLS_SANDBOX_DIR, +) -> ResolvedSkills: + """Resolve sample-bundled skills first, then user-configured skills.""" + sample_skills = skills_from_sample_metadata(sample_metadata) + if sample_skills.enabled: + return sample_skills + if not config_skills_dir: + return ResolvedSkills() + + host_dir = Path(config_skills_dir).expanduser() + if not host_dir.is_dir(): + raise FileNotFoundError(f'skills_dir is not a directory: {host_dir}') + + skills, errors = discover_skills(host_dir, path_prefix=prompt_base_dir) + return ResolvedSkills( + enabled=bool(skills), + source=CONFIG_SKILL_SOURCE, + host_dir=str(host_dir), + sandbox_dir=sandbox_dir, + prompt_base_dir=prompt_base_dir, + install_paths=list(install_paths), + skills=skills, + metadata_errors=errors, + ) + + +async def install_agent_skills( + environment: 'AgentEnvironment', + skills: ResolvedSkills, + *, + install_paths: Iterable[str] | None = None, + runner_name: str, + timeout: float = 60, +) -> None: + """Stage and install resolved skills into paths visible to an agent.""" + if not skills.enabled: + return + await stage_agent_skills(environment, skills, runner_name=runner_name) + + resolved_install_paths = _dedupe_paths(list(install_paths) if install_paths is not None else skills.install_paths) + command = install_skills_command(skills.sandbox_dir or '', resolved_install_paths) + if not command: + return + result = await environment.exec(['bash', '-lc', command], timeout=timeout) + if result.returncode != 0: + detail = ((result.stderr or result.stdout or '').strip() or f'rc={result.returncode}')[-1000:] + raise RuntimeError(f'{runner_name} failed to install skills: {detail}') + + +async def stage_agent_skills(environment: 'AgentEnvironment', skills: ResolvedSkills, *, runner_name: str) -> None: + """Upload host-configured skills into the environment when needed.""" + if not skills.enabled or skills.source != CONFIG_SKILL_SOURCE: + return + if not skills.host_dir or not skills.sandbox_dir: + raise RuntimeError(f'{runner_name} received config skills without host_dir and sandbox_dir') + try: + await environment.put_dir(skills.host_dir, skills.sandbox_dir) + except NotImplementedError as exc: + raise RuntimeError(f'{runner_name} requires environment.put_dir to install skills_dir') from exc + except Exception as exc: + raise RuntimeError(f'{runner_name} failed to stage skills: {exc}') from exc + + +def install_skills_command(source_dir: str, install_paths: List[str]) -> str | None: + """Return a POSIX shell command copying skills into discovery paths.""" + if not source_dir or not install_paths: + return None + commands = [] + quoted_source = shlex.quote(source_dir.rstrip('/')) + for dest in install_paths: + quoted_dest = quote_path_with_home(dest.rstrip('/')) + commands.append(f'mkdir -p {quoted_dest} && cp -R {quoted_source}/. {quoted_dest}/') + return ' && '.join(commands) + + +def quote_path_with_home(path: str) -> str: + """Quote a shell path while preserving leading ``$HOME`` expansion.""" + if path == '$HOME': + return '"$HOME"' + if path.startswith('$HOME/'): + rest = path[len('$HOME/'):] + if not rest: + return '"$HOME"' + return f'"$HOME"/{shlex.quote(rest)}' + return shlex.quote(path) + + +def _dedupe_paths(paths: Iterable[str]) -> List[str]: + resolved: List[str] = [] + seen = set() + for path in paths: + if not path or path in seen: + continue + seen.add(path) + resolved.append(path) + return resolved + + +__all__ = [ + 'CONFIG_SKILL_SOURCE', + 'DEFAULT_SKILLS_INSTALL_DIR', + 'DEFAULT_SKILLS_SANDBOX_DIR', + 'ResolvedSkills', + 'SkillMetadata', + 'TASK_BUNDLED_SKILL_SOURCE', + 'discover_skills', + 'format_skills_prompt', + 'install_agent_skills', + 'install_skills_command', + 'parse_frontmatter', + 'quote_path_with_home', + 'resolve_agent_skills', + 'stage_agent_skills', + 'skills_from_sample_metadata', +] diff --git a/evalscope/evalscope/agent/tools/bash.py b/evalscope/evalscope/agent/tools/bash.py index 99e21e9..2e488aa 100644 --- a/evalscope/evalscope/agent/tools/bash.py +++ b/evalscope/evalscope/agent/tools/bash.py @@ -1,6 +1,6 @@ """bash shell tool.""" -from typing import Optional +from typing import Any, Dict, List, Optional, Tuple from evalscope.api.agent import AgentEnvironment from evalscope.api.agent.types import ExecResult @@ -41,6 +41,56 @@ async def run_bash(call: ToolCall, env: Optional[AgentEnvironment]) -> str: return _format_exec_result(result) +def apply_bash_command_timeout_defaults( + handlers: Dict[str, Any], + tools: List[ToolInfo], + command_timeout: Optional[float], +) -> Tuple[Dict[str, Any], List[ToolInfo]]: + """Apply a native runtime default timeout to bash calls and schema.""" + if command_timeout is None: + return handlers, tools + updated_handlers = _apply_bash_command_timeout(handlers, command_timeout) + updated_tools = _apply_bash_tool_timeout_default(tools, command_timeout) + return updated_handlers, updated_tools + + +def _apply_bash_command_timeout(handlers: Dict[str, Any], command_timeout: float) -> Dict[str, Any]: + if 'bash' not in handlers: + return handlers + + bash_handler = handlers['bash'] + + async def run_bash_with_default_timeout(call: ToolCall, env: Optional[AgentEnvironment]) -> str: + args = dict(call.function.arguments or {}) + if 'timeout' not in args: + call = call.model_copy( + update={ + 'function': call.function.model_copy(update={'arguments': { + **args, + 'timeout': command_timeout, + }}) + } + ) + return await bash_handler(call, env) + + return {**handlers, 'bash': run_bash_with_default_timeout} + + +def _apply_bash_tool_timeout_default(tools: List[ToolInfo], command_timeout: float) -> List[ToolInfo]: + updated_tools: List[ToolInfo] = [] + for tool in tools: + if tool.name != 'bash': + updated_tools.append(tool) + continue + copied = tool.model_copy(deep=True) + timeout_param = copied.parameters.properties.get('timeout') + if timeout_param is not None: + timeout_param.default = command_timeout + timeout_param.description = f'Maximum execution time in seconds (default: {command_timeout:g}).' + updated_tools.append(copied) + return updated_tools + + def _format_exec_result(result: ExecResult) -> str: parts = [] if result.stdout: @@ -54,4 +104,4 @@ def _format_exec_result(result: ExecResult) -> str: return '\n'.join(parts) if parts else '(no output)' -__all__ = ['run_bash', 'BASH_TOOL_INFO'] +__all__ = ['run_bash', 'BASH_TOOL_INFO', 'apply_bash_command_timeout_defaults'] diff --git a/evalscope/evalscope/api/agent/environment.py b/evalscope/evalscope/api/agent/environment.py index e24d5f5..39391fb 100644 --- a/evalscope/evalscope/api/agent/environment.py +++ b/evalscope/evalscope/api/agent/environment.py @@ -7,6 +7,7 @@ live in ``evalscope/agent/environments/``. """ from abc import ABC, abstractmethod +from pathlib import Path from typing import Dict, List, Optional from .types import ExecResult @@ -46,6 +47,16 @@ class AgentEnvironment(ABC): """Release any external resources (containers, temp dirs, ...).""" ... + async def put_dir(self, source_dir: str | Path, target_dir: str) -> None: + """Copy a host directory into the environment. + + Environments that do not expose a writable filesystem bridge should + raise ``NotImplementedError``. The default keeps older custom + environments source-compatible while allowing runners to probe for the + capability when they need host-side assets such as Agent Skills. + """ + raise NotImplementedError(f'{type(self).__name__} does not support put_dir') + async def __aenter__(self) -> 'AgentEnvironment': return self diff --git a/evalscope/evalscope/api/agent/runner.py b/evalscope/evalscope/api/agent/runner.py index b8c8725..75516cf 100644 --- a/evalscope/evalscope/api/agent/runner.py +++ b/evalscope/evalscope/api/agent/runner.py @@ -44,12 +44,13 @@ def run_agent_loop( trace_strategy_name: Optional[str], trace_env_name: Optional[str], mcp_configs: Optional[List['MCPServerConfig']] = None, + close_environment: bool = True, ) -> AgentLoopResult: """Drive a single :class:`AgentLoop` to completion and return its result. - The environment (when provided) is closed in a ``finally`` block so - callers do not have to handle teardown themselves. ``AsyncioLoopRunner`` - bridges the async loop into a synchronous call site. + The environment (when provided) is closed in a ``finally`` block by + default so callers do not have to handle teardown themselves. + ``AsyncioLoopRunner`` bridges the async loop into a synchronous call site. Args: model: The :class:`Model` driving generation. @@ -66,6 +67,9 @@ def run_agent_loop( tools are merged into ``handlers`` / ``all_tools`` for the duration of the loop. Servers are spawned per sample (see :func:`evalscope.api.agent.mcp.resolve_mcp_tools`). + close_environment: Whether this helper owns and closes ``environment``. + Set to ``False`` when the caller needs to reuse the same + environment after the agent loop, for example to run a verifier. Returns: AgentLoopResult: Completed result with ``messages``, ``trace`` and @@ -110,7 +114,7 @@ def run_agent_loop( ) return await loop.run(ctx) finally: - if environment is not None: + if close_environment and environment is not None: await environment.close() return AsyncioLoopRunner.run(_run()) diff --git a/evalscope/evalscope/api/agent/types.py b/evalscope/evalscope/api/agent/types.py index faa42cc..dd506aa 100644 --- a/evalscope/evalscope/api/agent/types.py +++ b/evalscope/evalscope/api/agent/types.py @@ -7,7 +7,7 @@ import from ``evalscope.api.agent`` to participate. """ from dataclasses import dataclass, field -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional from evalscope.api.messages import ChatMessage @@ -43,15 +43,21 @@ class BaseAgentConfig(BaseModel): """Free-form variant-specific options. Native: forwarded to the strategy constructor. External: forwarded to the runner constructor.""" + skills_dir: Optional[str] = Field(default=None) + """Optional Agent Skills directory. Only agent runners consume this field.""" + + skill_prompt_nudge: bool = Field(default=True) + """Whether to add a neutral prompt nudge when skills are available.""" + class NativeAgentConfig(BaseAgentConfig): """AgentLoop-driven agent configuration. When carried by ``TaskConfig.agent_config``, every ``DefaultDataAdapter``-based benchmark routes inference through the - AgentLoop instead of calling ``model.generate`` once. Individual - AgentAdapter subclasses (e.g. SWE-bench_Pro) ignore this global config - and use their own settings to avoid double wrapping. + AgentLoop instead of calling ``model.generate`` once. AgentLoopAdapter + subclasses keep their benchmark defaults when fields are omitted and + accept explicitly configured strategy, tools and max_steps values. """ mode: Literal['native'] = Field(default='native') @@ -66,6 +72,13 @@ class NativeAgentConfig(BaseAgentConfig): max_steps: int = Field(default=10) """Hard upper bound on loop iterations.""" + command_timeout: Optional[float] = Field(default=None) + """Default timeout in seconds for native command-style tools. + + Tool calls that explicitly pass a timeout keep their own value. ``None`` + means use each tool's built-in default. + """ + mcp_servers: List[MCPServerConfig] = Field(default_factory=list) """List of MCP servers spawned alongside this agent's per-sample loop. @@ -76,6 +89,20 @@ class NativeAgentConfig(BaseAgentConfig): ``mcp_servers`` (the default) do not require ``pip install mcp``. """ + @field_validator('max_steps') + @classmethod + def _validate_max_steps(cls, v: int) -> int: + if v <= 0: + raise ValueError('max_steps must be greater than 0.') + return v + + @field_validator('command_timeout') + @classmethod + def _validate_command_timeout(cls, v: Optional[float]) -> Optional[float]: + if v is not None and v <= 0: + raise ValueError('command_timeout must be greater than 0.') + return v + class ExecResult(BaseModel): """Result of executing a command in an ``AgentEnvironment``.""" diff --git a/evalscope/evalscope/api/benchmark/adapters/agent_adapter.py b/evalscope/evalscope/api/benchmark/adapters/agent_adapter.py index 7e7f5e3..2882d57 100644 --- a/evalscope/evalscope/api/benchmark/adapters/agent_adapter.py +++ b/evalscope/evalscope/api/benchmark/adapters/agent_adapter.py @@ -19,6 +19,11 @@ Two extension modes are supported: from typing import TYPE_CHECKING, Any, Dict, List, Optional +from evalscope.agent.tools.bash import apply_bash_command_timeout_defaults +from evalscope.api.agent import AgentLoopResult, NativeAgentConfig +from evalscope.api.evaluator import InferenceResult +from evalscope.api.messages import ChatMessageUser +from evalscope.api.registry import get_strategy, resolve_tool_infos, resolve_tools from .default_data_adapter import DefaultDataAdapter if TYPE_CHECKING: @@ -52,16 +57,18 @@ class AgentLoopAdapter(AgentAdapter): #: ``'swe_bench_toolcall'``). strategy_name: str = 'function_calling' - #: Default upper bound on loop iterations per sample. Subclasses may - #: override either via class attribute or by pushing a ``max_steps`` - #: entry into ``extra_params`` (read in ``__init__``). + #: Optional benchmark-level default timeout for bash-style tools. Explicit + #: ``NativeAgentConfig.command_timeout`` values take precedence. + command_timeout_default: Optional[float] = None + + #: Default upper bound on loop iterations per sample. Subclasses override + #: this class attribute; users can explicitly override it through + #: ``NativeAgentConfig.max_steps``. max_steps_default: int = 30 def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) - # Allow benchmarks to expose ``max_steps`` to end users via the - # ``extra_params`` block of their ``BenchmarkMeta`` registration. - self.max_steps = int(self.extra_params.get('max_steps', self.max_steps_default)) + self.max_steps = self.max_steps_default # ------------------------------------------------------------------ # Build hooks @@ -73,8 +80,6 @@ class AgentLoopAdapter(AgentAdapter): Default: lookup ``self.strategy_name`` in the strategy registry and instantiate with default parameters. """ - from evalscope.api.registry import get_strategy - strategy_cls = get_strategy(self.strategy_name) return strategy_cls() @@ -90,6 +95,27 @@ class AgentLoopAdapter(AgentAdapter): """Return an :class:`AgentEnvironment` or ``None`` if not needed.""" return None + def _task_sandbox_config(self) -> Dict[str, Any]: + """Return task-level sandbox defaults for benchmark-owned environments.""" + if self._task_config is None or self._task_config.sandbox is None: + return {} + return dict(self._task_config.sandbox.default_config or {}) + + def _native_command_timeout(self) -> Optional[float]: + """Return the NativeAgentConfig command timeout, when explicitly configured.""" + if self._task_config is None: + return None + + ac = self._task_config.agent_config + if isinstance(ac, NativeAgentConfig): + return ac.command_timeout + return None + + def _resolve_command_timeout(self, ac: Any) -> Optional[float]: + if isinstance(ac, NativeAgentConfig) and 'command_timeout' in ac.model_fields_set: + return ac.command_timeout + return self.command_timeout_default + def build_initial_messages(self, sample: Any) -> List[Any]: """Return the message list the loop starts with. @@ -97,12 +123,90 @@ class AgentLoopAdapter(AgentAdapter): :class:`ChatMessageUser` when it is a plain string, otherwise copy the list. """ - from evalscope.api.messages import ChatMessageUser - if isinstance(sample.input, list): return list(sample.input) return [ChatMessageUser(content=sample.input)] + def build_max_steps_finalization_message(self, sample: Any) -> Optional[str]: + """Return a no-tools finalization prompt after the loop exhausts its step budget. + + The default ``None`` keeps the standard AgentLoop result. Benchmarks whose + official protocol requires one final model turn can override this hook. + """ + return None + + def should_finalize_after_max_steps(self, result: AgentLoopResult) -> bool: + """Return whether a max-steps result needs the optional final model turn.""" + return not result.final_output.completion.strip() + + def _maybe_run_external_agent(self, ac: Any, model: Any, sample: Any) -> Optional[InferenceResult]: + if ac is None or isinstance(ac, NativeAgentConfig): + return None + + # Local import to keep the bridge stack out of the adapter's + # module-load-time imports (no aiohttp dependency for non-external + # benchmark runs). + from evalscope.agent.external.adapter import run_external_agent + from evalscope.agent.external.config import ExternalAgentConfig + + if not isinstance(ac, ExternalAgentConfig): + return None + + messages = self.build_initial_messages(sample) + instruction = '\n\n'.join(m.text for m in messages if getattr(m, 'text', '')) + return run_external_agent( + config=ac, + model=model, + sample=sample, + environment_override=self.build_environment(sample), + instruction_override=instruction, + post_run_hook=self._external_extract_prediction, + ) + + def _resolve_strategy(self, sample: Any, ac: Any) -> Any: + strategy = self.build_strategy(sample) + if not isinstance(ac, NativeAgentConfig): + return strategy + + explicit_fields = ac.model_fields_set + if 'strategy' not in explicit_fields and 'kwargs' not in explicit_fields: + return strategy + + # Keep the benchmark strategy name when the user only supplies kwargs. + strategy_name = ac.strategy if 'strategy' in explicit_fields else strategy.name + return get_strategy(strategy_name)(**ac.kwargs) + + def _resolve_max_steps(self, ac: Any) -> int: + if isinstance(ac, NativeAgentConfig) and 'max_steps' in ac.model_fields_set: + return ac.max_steps + return self.max_steps + + def _resolve_tools(self, sample: Any, ac: Any) -> tuple[Dict[str, Any], List[Any]]: + handlers = self.build_tools(sample) + all_tools = list(sample.tools or []) + command_timeout = self._resolve_command_timeout(ac) + if not isinstance(ac, NativeAgentConfig): + return apply_bash_command_timeout_defaults(handlers, all_tools, command_timeout) + + if ac.tools: + configured_handlers = resolve_tools(ac.tools) + # Benchmark handlers win name collisions so required task semantics + # cannot be replaced by a global config. + handlers = {**configured_handlers, **handlers} + existing_tool_names = {tool.name for tool in all_tools} + for tool_info in resolve_tool_infos(ac.tools): + if tool_info.name not in existing_tool_names: + all_tools.append(tool_info) + existing_tool_names.add(tool_info.name) + + return apply_bash_command_timeout_defaults(handlers, all_tools, command_timeout) + + @staticmethod + def _resolve_mcp_configs(ac: Any) -> Optional[List['MCPServerConfig']]: + if isinstance(ac, NativeAgentConfig): + return ac.mcp_servers or None + return None + # ------------------------------------------------------------------ # Overridden inference hook # ------------------------------------------------------------------ @@ -110,11 +214,11 @@ class AgentLoopAdapter(AgentAdapter): def _on_inference(self, model: Any, sample: Any) -> Any: """Drive :class:`AgentLoop` for this sample and return the final output. - ``NativeAgentConfig.mcp_servers`` (if any) is forwarded to - :func:`run_agent_loop` so MCP-advertised tools merge into this - adapter's tool set without any benchmark-side change. Other - ``NativeAgentConfig`` fields are ignored — agentic benchmarks - are self-contained by design. + Benchmark defaults remain authoritative when ``agent_config`` is + omitted. Explicit ``NativeAgentConfig`` strategy, tools and max_steps + fields override or extend those defaults; MCP tools are always merged. + The benchmark keeps ownership of its environment so task-specific + mounts and sandbox contracts remain intact. :class:`ExternalAgentConfig` routes through :func:`run_external_agent` directly, with the adapter's :meth:`build_environment` and @@ -122,34 +226,20 @@ class AgentLoopAdapter(AgentAdapter): and prompt, and :meth:`_external_extract_prediction` recovering the prediction artifact before the env closes. """ - from evalscope.api.agent import AgentLoopResult, run_agent_loop - from evalscope.api.evaluator import InferenceResult + from evalscope.api.agent import run_agent_loop ac = self._task_config.agent_config if self._task_config is not None else None - mcp_configs: Optional[List['MCPServerConfig']] = None - if ac is not None: - # Local import to keep the bridge stack out of the adapter's - # module-load-time imports (no aiohttp dependency for non- - # external benchmark runs). - from evalscope.agent.external.adapter import run_external_agent - from evalscope.agent.external.config import ExternalAgentConfig - if isinstance(ac, ExternalAgentConfig): - messages = self.build_initial_messages(sample) - instruction = '\n\n'.join(m.text for m in messages if getattr(m, 'text', '')) - return run_external_agent( - config=ac, - model=model, - sample=sample, - environment_override=self.build_environment(sample), - instruction_override=instruction, - post_run_hook=self._external_extract_prediction, - ) - # NativeAgentConfig: forward only ``mcp_servers``; benchmark - # adapters own their strategy / tools / max_steps. - mcp_configs = ac.mcp_servers or None + external_result = self._maybe_run_external_agent(ac, model, sample) + if external_result is not None: + return external_result - strategy = self.build_strategy(sample) - handlers = self.build_tools(sample) + strategy = self._resolve_strategy(sample, ac) + handlers, all_tools = self._resolve_tools(sample, ac) + max_steps = self._resolve_max_steps(ac) + mcp_configs = self._resolve_mcp_configs(ac) + + if max_steps <= 0: + raise ValueError('AgentLoop max_steps must be greater than 0.') environment = self.build_environment(sample) result: AgentLoopResult = run_agent_loop( @@ -158,14 +248,18 @@ class AgentLoopAdapter(AgentAdapter): handlers=handlers, environment=environment, initial_messages=self.build_initial_messages(sample), - all_tools=list(sample.tools or []), - max_steps=self.max_steps, + all_tools=all_tools, + max_steps=max_steps, sample_id=sample.id, trace_strategy_name=getattr(strategy, 'name', None), trace_env_name=environment.name if environment else None, mcp_configs=mcp_configs, ) + finalization_prompt = self.build_max_steps_finalization_message(sample) + if finalization_prompt and self._reached_max_steps(result) and self.should_finalize_after_max_steps(result): + return self._finalize_after_max_steps(model, result, finalization_prompt) + # Resolve the final prediction through the strategy → adapter hook # chain so benchmarks (e.g. SWE-bench) can extract a custom payload # like a git patch from the trajectory. @@ -175,6 +269,63 @@ class AgentLoopAdapter(AgentAdapter): output.completion = final_text return InferenceResult(output=output, messages=result.messages, trace=result.trace) + @staticmethod + def _reached_max_steps(result: AgentLoopResult) -> bool: + if result.trace is None: + return False + from evalscope.api.agent import EventType + return any( + event.type == EventType.ERROR and event.payload.get('message') == 'max_steps_exceeded' + for event in result.trace.events + ) + + @staticmethod + def _finalize_after_max_steps(model: Any, result: AgentLoopResult, prompt: str) -> InferenceResult: + from evalscope.api.agent import EventType + + finalization_message = ChatMessageUser(content=prompt) + finalization_input = list(result.messages) + [finalization_message] + final_output = model.generate(input=finalization_input, tools=None) + messages = finalization_input + [final_output.message] + + step = result.trace.max_steps + result.trace.add_event( + step=step, + type=EventType.NUDGE, + message_id=finalization_message.id, + payload={'reason': 'max_steps_finalization'}, + ) + usage = None + if final_output.usage is not None: + usage = { + 'input': final_output.usage.input_tokens, + 'output': final_output.usage.output_tokens, + 'total': final_output.usage.total_tokens, + } + result.trace.add_event( + step=step, + type=EventType.MODEL_GENERATE, + message_id=final_output.message.id, + token_usage=usage, + payload={ + 'stop_reason': final_output.stop_reason, + 'phase': 'max_steps_finalization' + }, + ) + if final_output.completion.strip(): + result.trace.add_event( + step=step, + type=EventType.SUBMIT, + message_id=final_output.message.id, + payload={ + 'final_answer': final_output.completion, + 'phase': 'max_steps_finalization' + }, + ) + if result.trace.total_usage is not None and final_output.usage is not None: + result.trace.total_usage += final_output.usage + return InferenceResult(output=final_output, messages=messages, trace=result.trace) + async def _external_extract_prediction( self, env: Any, diff --git a/evalscope/evalscope/api/benchmark/adapters/default_data_adapter.py b/evalscope/evalscope/api/benchmark/adapters/default_data_adapter.py index 4d79150..fa9512b 100644 --- a/evalscope/evalscope/api/benchmark/adapters/default_data_adapter.py +++ b/evalscope/evalscope/api/benchmark/adapters/default_data_adapter.py @@ -253,7 +253,7 @@ class DefaultDataAdapter(DataAdapter): sample_fields=self.record_to_sample, # Custom sample conversion function filter_func=self.sample_filter, limit=self.limit if not self.reformat_subset else None, # Limit number of samples if specified - repeats=self.repeats, # Number of repetitions for each sample + repeats=1 if self.reformat_subset else self.repeats, # Number of repetitions for each sample shuffle=self.shuffle, # Shuffle dataset if enabled shuffle_choices=self.shuffle_choices, # Shuffle choices if requested data_source=self.dataset_hub, # Data source configuration @@ -878,7 +878,9 @@ class DefaultDataAdapter(DataAdapter): def finalize(self, *args, **kwargs): # Finalize the evaluation process - self.sandbox_finalize(*args, **kwargs) + sandbox_finalize = getattr(self, 'sandbox_finalize', None) + if callable(sandbox_finalize): + sandbox_finalize(*args, **kwargs) # Release dataset memory after evaluation to avoid accumulation across benchmarks self.test_dataset = None self.fewshot_dataset = None diff --git a/evalscope/evalscope/api/benchmark/benchmark.py b/evalscope/evalscope/api/benchmark/benchmark.py index 0376cf3..ad26cf4 100644 --- a/evalscope/evalscope/api/benchmark/benchmark.py +++ b/evalscope/evalscope/api/benchmark/benchmark.py @@ -9,7 +9,7 @@ from evalscope.api.dataset import DatasetDict, Sample from evalscope.api.evaluator import TaskState from evalscope.api.filter import FilterEnsemble, build_filter_ensemble from evalscope.api.metric import AggScore, SampleScore -from evalscope.api.mixin import LLMJudgeMixin, SandboxMixin +from evalscope.api.mixin import LLMJudgeMixin from evalscope.api.model import Model from evalscope.report import Report from evalscope.utils.logger import get_logger @@ -21,7 +21,7 @@ if TYPE_CHECKING: logger = get_logger() -class DataAdapter(LLMJudgeMixin, SandboxMixin, ABC): +class DataAdapter(LLMJudgeMixin, ABC): """ Data Adapter for the benchmark. """ diff --git a/evalscope/evalscope/api/benchmark/statistics.py b/evalscope/evalscope/api/benchmark/statistics.py index 4ae06af..f6a25b6 100644 --- a/evalscope/evalscope/api/benchmark/statistics.py +++ b/evalscope/evalscope/api/benchmark/statistics.py @@ -546,7 +546,7 @@ class SampleExample: # Check if truncation is needed truncated_data = truncate_value(raw_data, max_length, max_list_items) truncated_str = str(truncated_data) - was_truncated = '[TRUNCATED]' in truncated_str + was_truncated = '[TRUNCATED' in truncated_str return cls( data=truncated_data, diff --git a/evalscope/evalscope/api/dataset/__init__.py b/evalscope/evalscope/api/dataset/__init__.py index 4e874ae..baff2d6 100644 --- a/evalscope/evalscope/api/dataset/__init__.py +++ b/evalscope/evalscope/api/dataset/__init__.py @@ -1,3 +1,9 @@ +from .builder import ( + build_dataset_dict_from_record_map, + build_dataset_from_records, + load_local_file_dataset, + resolve_snapshot_or_local_path, +) from .dataset import Dataset, DatasetDict, FieldSpec, MemoryDataset, Sample -from .hub import DatasetHub, download_dataset_file, load_dataset_from_hub +from .hub import DatasetHub, download_dataset_file, download_dataset_snapshot, load_dataset_from_hub from .loader import DataLoader, DictDataLoader, LocalDataLoader, RemoteDataLoader diff --git a/evalscope/evalscope/api/dataset/builder.py b/evalscope/evalscope/api/dataset/builder.py new file mode 100644 index 0000000..08c9218 --- /dev/null +++ b/evalscope/evalscope/api/dataset/builder.py @@ -0,0 +1,123 @@ +import copy +from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Union + +from .dataset import DatasetDict, FieldSpec, MemoryDataset, Sample +from .hub import DatasetHub +from .loader import LocalDataLoader, _shuffle_in_place +from .utils import data_to_samples, record_to_sample_fn + +if TYPE_CHECKING: + from evalscope.api.benchmark import DataAdapter + + +def build_dataset_from_records( + records: Iterable[dict], + sample_fields: Union[FieldSpec, Callable], + *, + name: str, + location: Optional[str], + limit: Optional[Union[int, float]], + repeats: int, + shuffle: bool, + seed: Optional[int], + filter_func: Optional[Callable[[Sample], bool]] = None, + auto_id: bool = True, +) -> MemoryDataset: + """Build a MemoryDataset from raw records using the standard adapter mechanics. + + Note: + ``repeats`` duplicates each *resulting sample* (not the raw record) ``repeats`` + times consecutively, then ``reindex`` groups them with ``group_size=repeats``. + This matches the single-sample-per-record adapters. If ``sample_fields`` maps one + record to multiple samples, the per-sample duplication order differs from record + level duplication, which callers relying on k-metric grouping should be aware of. + """ + record_list = list(records) + if shuffle: + _shuffle_in_place(record_list, seed) + + if limit is not None: + if isinstance(limit, float): + if not (0.0 <= limit <= 1.0): + raise ValueError('Limit must be a non-negative integer or a float between 0 and 1.') + limit = int(len(record_list) * limit) + elif isinstance(limit, int) and limit < 0: + raise ValueError('Limit must be a non-negative integer or a float between 0 and 1.') + record_list = record_list[:limit] + + data_to_sample = record_to_sample_fn(sample_fields) + samples = data_to_samples(data=record_list, data_to_sample=data_to_sample) + if repeats > 1: + samples = [copy.deepcopy(sample) for sample in samples for _ in range(repeats)] + + dataset = MemoryDataset(samples=samples, name=name, location=location, shuffled=shuffle) + if filter_func is not None: + dataset = dataset.filter(filter_func) + if auto_id: + dataset.reindex(group_size=repeats if repeats > 0 else 1) + return dataset + + +def build_dataset_dict_from_record_map( + record_map: Dict[str, Iterable[dict]], + sample_fields: Union[FieldSpec, Callable], + *, + location: Optional[str], + limit: Optional[Union[int, float]], + repeats: int, + shuffle: bool, + seed: Optional[int], + filter_func: Optional[Callable[[Sample], bool]] = None, + auto_id: bool = True, +) -> DatasetDict: + """Build a DatasetDict from a mapping of subset name to raw records.""" + datasets = {} + for subset, records in record_map.items(): + datasets[subset] = build_dataset_from_records( + records=records, + sample_fields=sample_fields, + name=subset, + location=location, + limit=limit, + repeats=repeats, + shuffle=shuffle, + seed=seed, + filter_func=filter_func, + auto_id=auto_id, + ) + return DatasetDict(datasets) + + +def resolve_snapshot_or_local_path( + adapter: 'DataAdapter', + allow_file_pattern: Optional[Union[str, List[str]]] = None, +) -> str: + """Resolve an adapter dataset_id as a local path or downloaded snapshot root.""" + return DatasetHub( + data_id_or_path=adapter.dataset_id, + data_source=adapter.dataset_hub, + force_redownload=adapter.force_redownload, + cache_dir=adapter.dataset_dir, + ).download_snapshot(allow_file_pattern=allow_file_pattern) + + +def load_local_file_dataset( + adapter: 'DataAdapter', + dataset_path: str, + subset: str, + split: str, + sample_fields: Union[FieldSpec, Callable], + limit: Optional[Union[int, float]], + repeats: int, + shuffle: bool, +) -> MemoryDataset: + """Load a local JSONL/CSV/TSV file or directory with the standard LocalDataLoader.""" + return LocalDataLoader( + data_id_or_path=dataset_path, + split=split, + subset=subset, + sample_fields=sample_fields, + limit=limit, + repeats=repeats, + shuffle=shuffle, + ).load() diff --git a/evalscope/evalscope/api/dataset/hub.py b/evalscope/evalscope/api/dataset/hub.py index 4b0048d..a840aa7 100644 --- a/evalscope/evalscope/api/dataset/hub.py +++ b/evalscope/evalscope/api/dataset/hub.py @@ -3,7 +3,7 @@ import inspect import os from dataclasses import dataclass -from typing import Optional +from typing import List, Optional, Union from evalscope.constants import HubType from evalscope.utils.logger import get_logger @@ -44,6 +44,27 @@ class DatasetHub: cache_dir=self.cache_dir, ) + def download_snapshot( + self, + allow_file_pattern: Optional[Union[str, List[str]]] = None, + ignore_file_pattern: Optional[Union[str, List[str]]] = None, + ) -> str: + return download_dataset_snapshot( + data_id_or_path=self.data_id_or_path, + data_source=self.data_source, + revision=self.revision, + force_redownload=self.force_redownload, + cache_dir=self.cache_dir, + allow_file_pattern=allow_file_pattern, + ignore_file_pattern=ignore_file_pattern, + ) + + +def _resolve_data_source(data_id_or_path: str, data_source: Optional[str]) -> str: + if data_source == HubType.LOCAL or os.path.exists(data_id_or_path): + return HubType.LOCAL + return data_source or HubType.MODELSCOPE + def load_dataset_from_hub( data_id_or_path: str, @@ -61,7 +82,7 @@ def load_dataset_from_hub( from modelscope import MsDataset from modelscope.utils.constant import DownloadMode as MSDownloadMode - data_source = data_source or HubType.MODELSCOPE + data_source = _resolve_data_source(data_id_or_path, data_source) hf_download_mode = None if not force_redownload else HFDownloadMode.FORCE_REDOWNLOAD ms_download_mode = None if not force_redownload else MSDownloadMode.FORCE_REDOWNLOAD @@ -69,7 +90,7 @@ def load_dataset_from_hub( load_kwargs = dict( dataset_name=data_id_or_path, split=split, - subset_name=subset, + subset_name=subset if subset != 'default' else None, trust_remote_code=trust_remote, **kwargs, ) @@ -113,7 +134,16 @@ def download_dataset_file( cache_dir: Optional[str] = None, ) -> str: """Download or resolve a single file from a dataset hub.""" - data_source = data_source or HubType.MODELSCOPE + data_source = _resolve_data_source(data_id_or_path, data_source) + + if data_source == HubType.LOCAL: + root_dir = os.path.realpath(data_id_or_path) + resolved_path = os.path.realpath(os.path.join(root_dir, file_path)) + if os.path.commonpath([root_dir, resolved_path]) != root_dir: + raise ValueError(f'Invalid dataset file path: {file_path}') + if not os.path.exists(resolved_path): + raise FileNotFoundError(f'Dataset file {file_path} was not found in {root_dir}.') + return resolved_path if data_source == HubType.HUGGINGFACE: from huggingface_hub import hf_hub_download @@ -133,19 +163,60 @@ def download_dataset_file( download_kwargs = {'allow_file_pattern': file_path} if revision: download_kwargs['revision'] = revision + if cache_dir: + download_kwargs['cache_dir'] = cache_dir snapshot_dir = dataset_snapshot_download(data_id_or_path, **download_kwargs) resolved_path = os.path.join(snapshot_dir, file_path) if not os.path.exists(resolved_path): raise FileNotFoundError(f'Dataset file {file_path} was not found in {snapshot_dir}.') return resolved_path + raise ValueError(f'Unsupported dataset hub: {data_source}') + + +def download_dataset_snapshot( + data_id_or_path: str, + data_source: Optional[str] = HubType.MODELSCOPE, + revision: Optional[str] = None, + force_redownload: bool = False, + cache_dir: Optional[str] = None, + allow_file_pattern: Optional[Union[str, List[str]]] = None, + ignore_file_pattern: Optional[Union[str, List[str]]] = None, +) -> str: + """Download or resolve a dataset snapshot root from a supported hub.""" + data_source = _resolve_data_source(data_id_or_path, data_source) + if data_source == HubType.LOCAL: - root_dir = os.path.abspath(data_id_or_path) - resolved_path = os.path.abspath(os.path.join(root_dir, file_path)) - if os.path.commonpath([root_dir, resolved_path]) != root_dir: - raise ValueError(f'Invalid dataset file path: {file_path}') - if not os.path.exists(resolved_path): - raise FileNotFoundError(f'Dataset file {file_path} was not found in {root_dir}.') - return resolved_path + root_dir = os.path.realpath(data_id_or_path) + if not os.path.isdir(root_dir): + raise FileNotFoundError(f'Local dataset directory was not found: {data_id_or_path}') + return root_dir + + if data_source == HubType.HUGGINGFACE: + from huggingface_hub import snapshot_download + + return snapshot_download( + repo_id=data_id_or_path, + repo_type='dataset', + revision=revision, + cache_dir=cache_dir, + force_download=force_redownload, + allow_patterns=allow_file_pattern, + ignore_patterns=ignore_file_pattern, + ) + + if data_source == HubType.MODELSCOPE: + from modelscope import dataset_snapshot_download + + download_kwargs = {} + if revision: + download_kwargs['revision'] = revision + if cache_dir: + download_kwargs['cache_dir'] = cache_dir + if allow_file_pattern is not None: + download_kwargs['allow_file_pattern'] = allow_file_pattern + if ignore_file_pattern is not None: + download_kwargs['ignore_file_pattern'] = ignore_file_pattern + return dataset_snapshot_download(data_id_or_path, **download_kwargs) raise ValueError(f'Unsupported dataset hub: {data_source}') diff --git a/evalscope/evalscope/api/evaluator/cache.py b/evalscope/evalscope/api/evaluator/cache.py index a454b2d..58e3965 100644 --- a/evalscope/evalscope/api/evaluator/cache.py +++ b/evalscope/evalscope/api/evaluator/cache.py @@ -274,6 +274,9 @@ class ModelResult(BaseModel): messages: List[ChatMessage] = [] """Chat messages exchanged during evaluation (for conversational models).""" + agent_trace: Optional[AgentTrace] = None + """Structured agent trajectory, when prediction used an agent runner.""" + metadata: Optional[Dict[str, Any]] = None """Additional metadata associated with the model result.""" @@ -308,6 +311,7 @@ class ModelResult(BaseModel): model=task_state.model, index=task_state.sample_id, messages=task_state.messages, + agent_trace=task_state.agent_trace, model_output=task_state.output, metadata=task_state.metadata if save_metadata else {}, ) @@ -335,13 +339,15 @@ class ModelResult(BaseModel): if self.metadata: sample.metadata.update(self.metadata) - return TaskState( + task_state = TaskState( model=self.model, sample=sample, messages=self.messages, output=ModelOutput.model_validate(self.model_output), completed=True, # Mark as completed since it was cached ) + task_state.agent_trace = self.agent_trace + return task_state def pretty_print(self) -> str: """ diff --git a/evalscope/evalscope/api/messages/content.py b/evalscope/evalscope/api/messages/content.py index 77cea37..3be45b2 100644 --- a/evalscope/evalscope/api/messages/content.py +++ b/evalscope/evalscope/api/messages/content.py @@ -80,8 +80,8 @@ class ContentVideo(ContentBase): video: str """Video file path, URL, or base64 encoded data URL.""" - format: Literal['mp4', 'mpeg', 'mov'] - """Format of video data ('mp4', 'mpeg', or 'mov')""" + format: Literal['mp4', 'mpeg', 'mov', 'avi'] + """Format of video data ('mp4', 'mpeg', 'mov', or 'avi')""" start: Optional[float] = Field(default=None) """Optional start time of the relevant video segment in seconds.""" diff --git a/evalscope/evalscope/api/mixin/__init__.py b/evalscope/evalscope/api/mixin/__init__.py index 375f235..99decd2 100644 --- a/evalscope/evalscope/api/mixin/__init__.py +++ b/evalscope/evalscope/api/mixin/__init__.py @@ -1,2 +1,2 @@ +from .code_execution_sandbox_mixin import CodeExecutionSandboxMixin from .llm_judge_mixin import LLMJudgeMixin -from .sandbox_mixin import SandboxMixin diff --git a/evalscope/evalscope/api/mixin/sandbox_mixin.py b/evalscope/evalscope/api/mixin/code_execution_sandbox_mixin.py similarity index 76% rename from evalscope/evalscope/api/mixin/sandbox_mixin.py rename to evalscope/evalscope/api/mixin/code_execution_sandbox_mixin.py index 9075618..055541e 100644 --- a/evalscope/evalscope/api/mixin/sandbox_mixin.py +++ b/evalscope/evalscope/api/mixin/code_execution_sandbox_mixin.py @@ -1,25 +1,26 @@ -"""Sandbox mixin – thin wrapper around :class:`SandboxService`. +"""Code execution sandbox mixin backed by :class:`SandboxService`. -Historically this module owned all ms_enclave integration. The manager -lifecycle, engine dispatch and docker image build logic have since been -moved to :mod:`evalscope.api.sandbox`. The mixin now only: +This mixin is for benchmarks that execute generated code during scoring +(``match_score`` / verifier logic). It owns a benchmark-level sandbox pool and +does not model a full per-sample agent environment. -* decides whether a benchmark wants the sandbox (``use_sandbox``); +The mixin only: + +* decides whether a benchmark wants code execution sandboxing (``use_sandbox``); * resolves the engine + sandbox config from ``TaskConfig`` + ``BenchmarkMeta``; * exposes a simple ``execute_code_in_sandbox`` facade for adapters. """ from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union from evalscope.api.sandbox import ( + DockerImageSpec, PoolHandle, SandboxEngine, build_and_acquire_pool_sync, - default_docker_build_context, - ensure_docker_image_built, merge_sandbox_config_dicts, - normalize_docker_build_context, + prepare_docker_image, resolve_engine, ) from evalscope.utils.function_utils import AsyncioLoopRunner, thread_safe @@ -34,11 +35,11 @@ if TYPE_CHECKING: logger = get_logger() -class SandboxBackend(ABC): - """Abstract base class for sandbox backends. +class CodeExecutionBackend(ABC): + """Abstract base class for code execution sandbox backends. Kept so alternative non-ms_enclave backends can be plugged in later. - Today there is a single implementation (:class:`EnclaveSandboxBackend`). + Today there is a single implementation (:class:`EnclaveCodeExecutionBackend`). """ def __init__(self, benchmark_meta: 'BenchmarkMeta', task_config: 'TaskConfig'): @@ -62,21 +63,19 @@ class SandboxBackend(ABC): """Release any sandbox-local resources (not the service itself).""" -class EnclaveSandboxBackend(SandboxBackend): - """ms_enclave-backed sandbox backend delegating to :class:`SandboxService`.""" +class EnclaveCodeExecutionBackend(CodeExecutionBackend): + """ms_enclave-backed code execution pool delegating to :class:`SandboxService`.""" def __init__( self, benchmark_meta: 'BenchmarkMeta', task_config: 'TaskConfig', - use_custom_image: bool = False, - build_context_provider: Optional[Callable[[], Tuple[str, str]]] = None, + image_spec_provider: Optional[Callable[[], Optional[DockerImageSpec]]] = None, ): super().__init__(benchmark_meta, task_config) self._pool_handle: Optional[PoolHandle] = None self._pool_size: int = self._resolve_pool_size() - self._use_custom_image = use_custom_image - self._build_context_provider = build_context_provider + self._image_spec_provider = image_spec_provider def _resolve_pool_size(self) -> int: if not self._task_config: @@ -96,10 +95,12 @@ class EnclaveSandboxBackend(SandboxBackend): manager_config = self._resolve_manager_config() if engine is SandboxEngine.DOCKER: - image = sandbox_cfg_dict.get('image') - if self._use_custom_image and image: - build_ctx, dockerfile = self._get_build_context() - ensure_docker_image_built(image, path=build_ctx, dockerfile=dockerfile, label='Sandbox image') + image_spec = self._image_spec_provider() if self._image_spec_provider is not None else None + if image_spec is not None: + result = prepare_docker_image(image_spec) + sandbox_cfg_dict = dict(sandbox_cfg_dict) + sandbox_cfg_dict['image'] = result.image_tag + logger.info(f'Sandbox image prepared: {result.image_tag} (reused={result.reused})') self._pool_handle = build_and_acquire_pool_sync( engine=engine, @@ -160,13 +161,6 @@ class EnclaveSandboxBackend(SandboxBackend): # Manager lifecycle is owned by SandboxService; we only drop our pool reference. self._pool_handle = None - def _get_build_context(self) -> Tuple[str, str]: - if self._build_context_provider is None: - build_ctx, dockerfile = default_docker_build_context() - else: - build_ctx, dockerfile = self._build_context_provider() - return normalize_docker_build_context(build_ctx, dockerfile) - # ------------------------------------------------------------------ # Config resolution helpers # ------------------------------------------------------------------ @@ -183,7 +177,7 @@ class EnclaveSandboxBackend(SandboxBackend): return merge_sandbox_config_dicts(meta_default, task_default) # ------------------------------------------------------------------ - # Legacy accessor used by ``SandboxMixin.sandbox_manager`` + # Accessor used by ``CodeExecutionSandboxMixin.sandbox_manager`` # ------------------------------------------------------------------ @property @@ -191,21 +185,21 @@ class EnclaveSandboxBackend(SandboxBackend): return self._pool_handle.manager if self._pool_handle else None -class SandboxMixin: - """Sandbox mixin for sandboxed code execution.""" +class CodeExecutionSandboxMixin: + """Mixin for benchmarks that execute generated code in a sandbox pool.""" def __init__(self, benchmark_meta: 'BenchmarkMeta', task_config: Optional['TaskConfig'] = None): self._benchmark_meta = benchmark_meta self._task_config = task_config - self._backend: Optional[SandboxBackend] = None - """Sandbox backend instance.""" + self._backend: Optional[CodeExecutionBackend] = None + """Code execution sandbox backend instance.""" - super().__init__() + super().__init__(benchmark_meta=benchmark_meta, task_config=task_config) @property def use_sandbox(self) -> bool: - """Return whether to use sandbox for the benchmark.""" + """Return whether to use sandboxed code execution for the benchmark.""" if not self._task_config or self._task_config.sandbox is None: return False return bool(self._task_config.sandbox.enabled) @@ -213,24 +207,23 @@ class SandboxMixin: @property def sandbox_manager(self) -> Optional['SandboxManager']: """Get the underlying SandboxManager instance (or ``None`` if not started).""" - if isinstance(self._backend, EnclaveSandboxBackend): + if isinstance(self._backend, EnclaveCodeExecutionBackend): return self._backend.manager return None - def _get_backend(self) -> SandboxBackend: + def _get_backend(self) -> CodeExecutionBackend: if self._backend: return self._backend - self._backend = EnclaveSandboxBackend( + self._backend = EnclaveCodeExecutionBackend( self._benchmark_meta, self._task_config, - use_custom_image=bool(getattr(self, '_use_custom_image', False)), - build_context_provider=self.get_build_context, + image_spec_provider=self.get_sandbox_image_spec, ) return self._backend - def get_build_context(self) -> Tuple[str, str]: - """Return Docker build context for benchmarks using a custom sandbox image.""" - return default_docker_build_context() + def get_sandbox_image_spec(self) -> Optional[DockerImageSpec]: + """Return a benchmark-level Docker image spec for the code execution pool, if needed.""" + return None @thread_safe def ensure_sandbox_ready(self) -> bool: diff --git a/evalscope/evalscope/api/mixin/llm_judge_mixin.py b/evalscope/evalscope/api/mixin/llm_judge_mixin.py index faac869..cd5a620 100644 --- a/evalscope/evalscope/api/mixin/llm_judge_mixin.py +++ b/evalscope/evalscope/api/mixin/llm_judge_mixin.py @@ -4,6 +4,7 @@ from evalscope.api.evaluator import TaskState from evalscope.api.metric import Score from evalscope.constants import JudgeStrategy from evalscope.metrics import LLMJudge +from evalscope.utils.argument_utils import get_secret_value from evalscope.utils.logger import get_logger if TYPE_CHECKING: @@ -18,17 +19,17 @@ class LLMJudgeMixin: Mixin class for LLM Judge functionality. """ + llm_judge_default = False + """Whether JudgeStrategy.AUTO should use LLM judge by default.""" + def __init__(self, benchmark_meta: 'BenchmarkMeta', task_config: Optional['TaskConfig'] = None): self._benchmark_meta = benchmark_meta self._task_config = task_config - self._use_llm_judge = False - """Whether to use LLM as a judge""" - self._llm_judge: Optional[LLMJudge] = None """LLM judge instance""" - super().__init__(benchmark_meta=benchmark_meta, task_config=task_config) + super().__init__() @property def llm_judge(self) -> Optional[LLMJudge]: @@ -57,7 +58,7 @@ class LLMJudgeMixin: elif self.judge_strategy == JudgeStrategy.LLM_RECALL: return True elif self.judge_strategy == JudgeStrategy.AUTO: - return self._use_llm_judge + return self.llm_judge_default else: logger.warning(f'Unknown judge strategy: {self.judge_strategy}. Defaulting to False.') return False @@ -78,7 +79,8 @@ class LLMJudgeMixin: 'LLM judge model arguments must be provided for LLM-based judge strategies. ' 'Please check your task configuration.' ) - return LLMJudge(**self._task_config.judge_model_args) + judge_model_args = get_secret_value(self._task_config.judge_model_args) + return LLMJudge(**judge_model_args) def maybe_llm_match_score( self, diff --git a/evalscope/evalscope/api/model/__init__.py b/evalscope/evalscope/api/model/__init__.py index b090921..45f41fc 100644 --- a/evalscope/evalscope/api/model/__init__.py +++ b/evalscope/evalscope/api/model/__init__.py @@ -1,4 +1,4 @@ -from .generate_config import GenerateConfig +from .generate_config import AnthropicCacheControl, GenerateConfig from .lazy_model import LazyModel from .model import Model, ModelAPI, get_model, get_model_with_task_config from .model_output import ( diff --git a/evalscope/evalscope/api/model/generate_config.py b/evalscope/evalscope/api/model/generate_config.py index b2a8865..f66f3af 100644 --- a/evalscope/evalscope/api/model/generate_config.py +++ b/evalscope/evalscope/api/model/generate_config.py @@ -1,8 +1,9 @@ # flake8: noqa: E501 from copy import deepcopy -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator, model_validator from typing import Any, Dict, List, Literal, Optional +from evalscope.utils.argument_utils import secretize_auth_headers from evalscope.utils.json_schema import JSONSchema @@ -23,6 +24,18 @@ class ResponseSchema(BaseModel): OpenAI and Mistral only.""" +class AnthropicCacheControl(BaseModel): + """Anthropic prompt cache control.""" + + model_config = {'extra': 'forbid'} + + type: Literal['ephemeral'] = Field(default='ephemeral') + """Anthropic cache type.""" + + ttl: Optional[Literal['5m', '1h']] = Field(default=None) + """Optional cache time-to-live.""" + + class GenerateConfig(BaseModel): """Model generation options.""" model_config = {'extra': 'allow'} @@ -102,6 +115,16 @@ class GenerateConfig(BaseModel): reasoning_tokens: Optional[int] = Field(default=None) """Maximum number of tokens to use for reasoning. Anthropic Claude models only.""" + anthropic_cache_control: Optional[AnthropicCacheControl] = Field(default=None) + """Anthropic prompt cache control. Set to ``{"type": "ephemeral"}`` to enable prompt caching.""" + + anthropic_cache_strategy: Literal['evaluation', 'recent_messages'] = Field(default='evaluation') + """Anthropic prompt cache breakpoint placement strategy. + + ``evaluation`` caches stable evaluation prefixes such as tools, system prompts, and few-shot examples. + ``recent_messages`` caches stable agent prefixes and the growing multi-turn conversation history. + """ + reasoning_summary: Optional[Literal['concise', 'detailed', 'auto']] = Field(default=None) """Provide summary of reasoning steps (defaults to no summary). Use 'auto' to access the most detailed summarizer available for the current model. OpenAI reasoning models only.""" @@ -132,7 +155,7 @@ class GenerateConfig(BaseModel): extra_query: Optional[Dict[str, Any]] = Field(default=None) """Extra query parameters to be sent with requests to OpenAI compatible servers. OpenAI, vLLM, and SGLang only.""" - extra_headers: Optional[Dict[str, str]] = Field(default=None) + extra_headers: Optional[Dict[str, Any]] = Field(default=None) """Extra headers to be sent with requests to OpenAI compatible servers. OpenAI, vLLM, and SGLang only.""" height: Optional[int] = Field(default=None) @@ -147,6 +170,21 @@ class GenerateConfig(BaseModel): guidance_scale: Optional[float] = Field(default=None) """Guidance scale for image generation model only""" + @model_validator(mode='before') + @classmethod + def reject_legacy_anthropic_cache_config(cls, data: Any) -> Any: + if isinstance(data, dict) and 'anthropic_content_cache_control' in data: + raise ValueError( + '`anthropic_content_cache_control` has been replaced by `anthropic_cache_control`. ' + 'Use `anthropic_cache_strategy` to choose breakpoint placement.' + ) + return data + + @field_validator('extra_headers', mode='after') + @classmethod + def _validate_extra_headers(cls, value: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + return secretize_auth_headers(value) + def merge(self, other: 'GenerateConfig') -> 'GenerateConfig': """Merge another model configuration into this one. diff --git a/evalscope/evalscope/api/model/model.py b/evalscope/evalscope/api/model/model.py index ad58f71..b0231e8 100644 --- a/evalscope/evalscope/api/model/model.py +++ b/evalscope/evalscope/api/model/model.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any, Dict, Generator, List, Literal, Optional, from evalscope.api.messages import ChatMessage, ChatMessageAssistant, ChatMessageSystem, ChatMessageUser from evalscope.api.registry import get_model_api from evalscope.api.tool import ToolChoice, ToolFunction, ToolInfo -from evalscope.utils import get_logger +from evalscope.utils import get_logger, get_secret_value from evalscope.utils.function_utils import thread_safe from .generate_config import GenerateConfig from .model_output import ModelOutput @@ -305,10 +305,7 @@ def get_model_with_task_config(task_config: 'TaskConfig') -> Model: model = task_config.model eval_type = task_config.eval_type base_url = task_config.api_url - api_key = task_config.api_key - # TaskConfig defaults api_key to 'EMPTY'; treat it as unset so env vars can be used. - if api_key == 'EMPTY': - api_key = None + api_key = get_secret_value(task_config.api_key) config = task_config.generation_config model_args = task_config.model_args or {} @@ -364,6 +361,10 @@ def get_model( if cached is not None: return cached + # TaskConfig defaults api_key to 'EMPTY'; treat it as unset so env vars can be used. + if api_key == 'EMPTY': + api_key = None + logger.info( f'Creating model {model} with eval_type={eval_type} ' f'base_url={base_url}, config={config.model_dump(exclude_none=True)}, model_args={model_args}' diff --git a/evalscope/evalscope/api/sandbox/__init__.py b/evalscope/evalscope/api/sandbox/__init__.py index 3cb2299..28d7fbc 100644 --- a/evalscope/evalscope/api/sandbox/__init__.py +++ b/evalscope/evalscope/api/sandbox/__init__.py @@ -1,7 +1,7 @@ """Unified sandbox service layer for evalscope. This package consolidates the ms_enclave integration used by both -:class:`evalscope.api.mixin.sandbox_mixin.SandboxMixin` (pool-based execution +:class:`evalscope.api.mixin.code_execution_sandbox_mixin.CodeExecutionSandboxMixin` (pool-based execution for code benchmarks) and the Agent environment (per-sample containers). Public surface: @@ -21,6 +21,13 @@ from .config_builder import ( normalize_docker_build_context, should_build_docker_image, ) +from .docker_image import ( + DockerImageBuilder, + DockerImageResult, + DockerImageSpec, + hash_build_context, + prepare_docker_image, +) from .engine import SandboxEngine, get_enclave_types, resolve_engine from .service import ( PoolHandle, @@ -39,12 +46,17 @@ __all__ = [ 'build_and_acquire_pool_sync', 'build_docker_image', 'build_sandbox_config', + 'DockerImageBuilder', + 'DockerImageResult', + 'DockerImageSpec', 'default_docker_build_context', 'ensure_docker_image_built', 'get_enclave_types', 'get_sandbox_service', + 'hash_build_context', 'merge_sandbox_config_dicts', 'normalize_docker_build_context', + 'prepare_docker_image', 'resolve_engine', 'shutdown_sandbox_service', 'should_build_docker_image', diff --git a/evalscope/evalscope/api/sandbox/docker_image.py b/evalscope/evalscope/api/sandbox/docker_image.py new file mode 100644 index 0000000..2be155b --- /dev/null +++ b/evalscope/evalscope/api/sandbox/docker_image.py @@ -0,0 +1,144 @@ +"""Small Docker image build/reuse helper. + +This module is intentionally independent from the ms_enclave sandbox service: +benchmarks can use it to prepare local images, then pass the resulting tag to +their existing sandbox/environment layer. +""" + +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +from pydantic import BaseModel, Field +from typing import Any, Dict, List + +from evalscope.utils.logger import get_logger +from .config_builder import build_docker_image, normalize_docker_build_context, should_build_docker_image + +logger = get_logger() + + +class DockerImageSpec(BaseModel): + """Description of a local Docker build.""" + + name_prefix: str + context_dir: str + dockerfile: str = 'Dockerfile' + build_args: Dict[str, str] = Field(default_factory=dict) + cache_key_parts: List[str] = Field(default_factory=list) + force_rebuild: bool = False + + +class DockerImageResult(BaseModel): + """Result returned by :class:`DockerImageBuilder`.""" + + image_tag: str + reused: bool + context_hash: str + + +class DockerImageBuilder: + """Build or reuse a local Docker image tagged from a content hash.""" + + builder_version: str = 'v1' + + def build_or_reuse(self, spec: DockerImageSpec) -> DockerImageResult: + context_dir, dockerfile = normalize_docker_build_context(spec.context_dir, spec.dockerfile) + context_hash = hash_build_context( + context_dir, + cache_key_parts=[self.builder_version, *spec.cache_key_parts, *_build_args_cache_parts(spec.build_args)], + ) + image_tag = f'{_sanitize_tag_part(spec.name_prefix)}:{context_hash[:24]}' + should_build = spec.force_rebuild or should_build_docker_image(image_tag) + if should_build: + logger.info(f'Docker image {image_tag!r} not found or rebuild requested. Building from {context_dir} ...') + build_docker_image_with_args( + image=image_tag, + path=context_dir, + dockerfile=dockerfile, + build_args=spec.build_args, + ) + logger.info(f'Docker image built: {image_tag}') + return DockerImageResult(image_tag=image_tag, reused=not should_build, context_hash=context_hash) + + +def prepare_docker_image(spec: DockerImageSpec, *, builder: DockerImageBuilder | None = None) -> DockerImageResult: + """Prepare a Docker image from ``spec`` using the shared local builder.""" + return (builder or DockerImageBuilder()).build_or_reuse(spec) + + +def hash_build_context(context_dir: str, *, cache_key_parts: List[str] | None = None) -> str: + """Return a stable hash for regular files under ``context_dir``.""" + root = Path(context_dir) + digest = hashlib.sha256() + for part in cache_key_parts or []: + digest.update(b'part\0') + digest.update(str(part).encode('utf-8')) + digest.update(b'\0') + for file_path in sorted(path for path in root.rglob('*') if path.is_file() and not path.is_symlink()): + rel = file_path.relative_to(root).as_posix() + digest.update(b'file\0') + digest.update(rel.encode('utf-8')) + digest.update(b'\0') + with file_path.open('rb') as f: + for chunk in iter(lambda: f.read(1024 * 1024), b''): + digest.update(chunk) + try: + mode = file_path.stat().st_mode & 0o777 + except OSError: + mode = 0 + digest.update(f'\0mode:{mode:o}'.encode('ascii')) + return digest.hexdigest() + + +def build_docker_image_with_args( + image: str, + path: str, + dockerfile: str = 'Dockerfile', + build_args: Dict[str, str] | None = None, +) -> Any: + """Build a Docker image with optional build args.""" + if not build_args: + return build_docker_image(image=image, path=path, dockerfile=dockerfile) + + from docker.client import DockerClient + + docker_client = DockerClient.from_env() + build_logs = docker_client.images.build( + path=path, + dockerfile=dockerfile, + tag=image, + rm=True, + buildargs=build_args, + ) + for log in build_logs[1]: + if 'stream' in log: + logger.info(log['stream'].strip()) + elif 'error' in log: + logger.error(log['error']) + return build_logs[0] + + +def _sanitize_tag_part(value: str) -> str: + chars = [] + for ch in value.lower(): + if ch.isalnum() or ch in '._-/': + chars.append(ch) + else: + chars.append('-') + text = ''.join(chars).strip('.-/') + return text or 'evalscope-image' + + +def _build_args_cache_parts(build_args: Dict[str, str]) -> List[str]: + return [f'build_arg:{key}={value}' for key, value in sorted(build_args.items())] + + +__all__ = [ + 'DockerImageBuilder', + 'DockerImageResult', + 'DockerImageSpec', + 'hash_build_context', + 'prepare_docker_image', +] diff --git a/evalscope/evalscope/api/sandbox/service.py b/evalscope/evalscope/api/sandbox/service.py index 109e56f..560007f 100644 --- a/evalscope/evalscope/api/sandbox/service.py +++ b/evalscope/evalscope/api/sandbox/service.py @@ -2,7 +2,7 @@ Unifies the two historical code paths: -* ``SandboxMixin.EnclaveSandboxBackend`` – one manager per benchmark, pooled. +* ``CodeExecutionSandboxMixin.EnclaveCodeExecutionBackend`` – one manager per benchmark, pooled. * ``EnclaveAgentEnvironment`` – one manager per process, per-sample containers. Both are now thin wrappers around :class:`SandboxService`. The service @@ -17,6 +17,7 @@ import asyncio import atexit import json import threading +from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from evalscope.utils.function_utils import AsyncioLoopRunner @@ -73,6 +74,12 @@ class SandboxHandle: raise RuntimeError('SandboxHandle already closed') return await self._manager.execute_tool(self._sandbox_id, tool_name, parameters) + async def put_dir(self, source_dir: str | Path, target_dir: str) -> bool: + """Copy a host directory into the sandbox via ms_enclave SandboxManager.""" + if self._sandbox_id is None: + raise RuntimeError('SandboxHandle already closed') + return await self._manager.put_dir(self._sandbox_id, source_dir, target_dir) + async def close(self) -> None: if self._sandbox_id is None: return @@ -197,7 +204,7 @@ class SandboxService: return SandboxManagerFactory.create_manager(**manager_config) # ------------------------------------------------------------------ - # Public APIs: pooled (SandboxMixin) and per-sample (Agent env) + # Public APIs: pooled (CodeExecutionSandboxMixin) and per-sample (Agent env) # ------------------------------------------------------------------ async def acquire_pool( @@ -241,6 +248,14 @@ class SandboxService: logger.info('SandboxService: manager stopped.') except Exception as exc: logger.warning(f'SandboxService: error stopping manager: {exc}') + cleanup_all = getattr(manager, 'cleanup_all_sandboxes', None) + if not callable(cleanup_all): + continue + try: + await cleanup_all() + logger.info('SandboxService: fallback sandbox cleanup completed.') + except Exception as cleanup_exc: + logger.warning(f'SandboxService: fallback sandbox cleanup failed: {cleanup_exc}') def shutdown_all(self) -> None: """Synchronous wrapper around :meth:`shutdown_all_async`.""" @@ -285,7 +300,7 @@ def shutdown_sandbox_service() -> None: # --------------------------------------------------------------------------- -# Convenience helpers used by SandboxMixin / EnclaveAgentEnvironment +# Convenience helpers used by CodeExecutionSandboxMixin / EnclaveAgentEnvironment # --------------------------------------------------------------------------- @@ -295,7 +310,7 @@ def build_and_acquire_pool_sync( sandbox_config_dict: Optional[Dict[str, Any]], manager_config: Optional[Dict[str, Any]] = None, ) -> PoolHandle: - """Synchronous helper for :class:`SandboxMixin`. + """Synchronous helper for :class:`CodeExecutionSandboxMixin`. Combines :func:`build_sandbox_config` and :meth:`SandboxService.acquire_pool` and drives them through the shared :class:`AsyncioLoopRunner`. diff --git a/evalscope/evalscope/benchmarks/_meta/a_okvqa.json b/evalscope/evalscope/benchmarks/_meta/a_okvqa.json index b3e8f7d..24024f2 100644 --- a/evalscope/evalscope/benchmarks/_meta/a_okvqa.json +++ b/evalscope/evalscope/benchmarks/_meta/a_okvqa.json @@ -158,4 +158,4 @@ }, "updated_at": "2026-01-28T17:31:32.179469", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/aa_lcr.json b/evalscope/evalscope/benchmarks/_meta/aa_lcr.json index 2a8216e..a05c042 100644 --- a/evalscope/evalscope/benchmarks/_meta/aa_lcr.json +++ b/evalscope/evalscope/benchmarks/_meta/aa_lcr.json @@ -82,4 +82,4 @@ }, "updated_at": "2026-01-28T17:31:32.178244", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/acebench.json b/evalscope/evalscope/benchmarks/_meta/acebench.json index 2d6d47a..3eb08a3 100644 --- a/evalscope/evalscope/benchmarks/_meta/acebench.json +++ b/evalscope/evalscope/benchmarks/_meta/acebench.json @@ -433,4 +433,4 @@ }, "updated_at": "2026-06-01T15:05:41.862420", "translation_updated_at": "2026-06-01T15:06:01" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/agieval.json b/evalscope/evalscope/benchmarks/_meta/agieval.json new file mode 100644 index 0000000..9e2dc5e --- /dev/null +++ b/evalscope/evalscope/benchmarks/_meta/agieval.json @@ -0,0 +1,286 @@ +{ + "meta": { + "pretty_name": "AGIEval", + "dataset_id": "opencompass/agieval", + "paper_url": null, + "tags": [ + "Reasoning", + "Knowledge", + "Math", + "MCQ" + ], + "metrics": [ + "acc" + ], + "few_shot_num": 0, + "eval_split": "test", + "train_split": "dev", + "subset_list": [ + "aqua-rat", + "logiqa-en", + "lsat-ar", + "lsat-lr", + "lsat-rc", + "sat-math", + "sat-en", + "sat-en-without-passage", + "gaokao-english", + "logiqa-zh", + "gaokao-chinese", + "gaokao-geography", + "gaokao-history", + "gaokao-biology", + "gaokao-chemistry", + "gaokao-physics", + "gaokao-mathqa", + "jec-qa-kd", + "jec-qa-ca", + "math", + "gaokao-mathcloze" + ], + "description": "\n## Overview\n\nAGIEval is a human-centric benchmark designed to evaluate foundation models in the context of human cognition and problem-solving. It uses official, standard, and authoritative admission and qualification exams intended for general human test-takers, such as college entrance exams (GaoKao), law school admission tests (LSAT), math competitions, and lawyer qualification exams.\n\n## Task Description\n\n- **Task Type**: Mixed (Multiple-Choice QA + Open-ended Math)\n- **Input**: Questions from standardized exams with optional passages and answer choices\n- **Output**: Answer letter(s) for MCQ, or numerical/mathematical answer for open-ended\n- **Languages**: English and Chinese\n\n## Key Features\n\n- 21 subsets covering diverse exam types across two languages\n- English MCQ: LSAT (AR/LR/RC), SAT (Math/English), AQuA-RAT, LogiQA, GaoKao-English\n- Chinese MCQ: GaoKao (Chinese/Geography/History/Biology/Chemistry/Physics/MathQA), LogiQA-zh, JEC-QA\n- Open-ended math: MATH (English), GaoKao-MathCloze (Chinese)\n- Multi-select subsets: JEC-QA-KD, JEC-QA-CA, GaoKao-Physics\n- Includes passage-based reading comprehension questions\n\n## Evaluation Notes\n\n- MCQ subsets use evalscope's standard MultiChoice template and extraction\n- Multi-select subsets use Chinese multi-answer template\n- Math/cloze subsets use mathematical equivalence checking\n- CoT (Chain-of-Thought) prompting enabled by default\n", + "prompt_template": "Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of {letters}. Think step by step before answering.\n\n{question}\n\n{choices}", + "system_prompt": "", + "few_shot_prompt_template": "", + "aggregation": "mean", + "extra_params": {}, + "sandbox_config": {}, + "category": "llm" + }, + "statistics": { + "total_samples": 8269, + "subset_stats": [ + { + "name": "aqua-rat", + "sample_count": 254, + "prompt_length_mean": 290.09, + "prompt_length_min": 103, + "prompt_length_max": 587, + "prompt_length_std": 84.38, + "target_length_mean": 1 + }, + { + "name": "logiqa-en", + "sample_count": 651, + "prompt_length_mean": 911.89, + "prompt_length_min": 248, + "prompt_length_max": 1769, + "prompt_length_std": 250.8, + "target_length_mean": 1 + }, + { + "name": "lsat-ar", + "sample_count": 230, + "prompt_length_mean": 946.36, + "prompt_length_min": 635, + "prompt_length_max": 1853, + "prompt_length_std": 183.41, + "target_length_mean": 1 + }, + { + "name": "lsat-lr", + "sample_count": 510, + "prompt_length_mean": 1156.66, + "prompt_length_min": 563, + "prompt_length_max": 2348, + "prompt_length_std": 261.19, + "target_length_mean": 1 + }, + { + "name": "lsat-rc", + "sample_count": 269, + "prompt_length_mean": 3652.86, + "prompt_length_min": 2959, + "prompt_length_max": 4825, + "prompt_length_std": 320.49, + "target_length_mean": 1 + }, + { + "name": "sat-math", + "sample_count": 220, + "prompt_length_mean": 392.45, + "prompt_length_min": 120, + "prompt_length_max": 1201, + "prompt_length_std": 232.93, + "target_length_mean": 1 + }, + { + "name": "sat-en", + "sample_count": 206, + "prompt_length_mean": 4618.28, + "prompt_length_min": 3569, + "prompt_length_max": 5316, + "prompt_length_std": 383.18, + "target_length_mean": 1 + }, + { + "name": "sat-en-without-passage", + "sample_count": 206, + "prompt_length_mean": 435.91, + "prompt_length_min": 169, + "prompt_length_max": 937, + "prompt_length_std": 145.26, + "target_length_mean": 1 + }, + { + "name": "gaokao-english", + "sample_count": 306, + "prompt_length_mean": 2025.44, + "prompt_length_min": 517, + "prompt_length_max": 4216, + "prompt_length_std": 565.35, + "target_length_mean": 1 + }, + { + "name": "logiqa-zh", + "sample_count": 651, + "prompt_length_mean": 267.62, + "prompt_length_min": 98, + "prompt_length_max": 526, + "prompt_length_std": 59.84, + "target_length_mean": 1 + }, + { + "name": "gaokao-chinese", + "sample_count": 246, + "prompt_length_mean": 988.09, + "prompt_length_min": 152, + "prompt_length_max": 2186, + "prompt_length_std": 492.07, + "target_length_mean": 1 + }, + { + "name": "gaokao-geography", + "sample_count": 199, + "prompt_length_mean": 204.82, + "prompt_length_min": 64, + "prompt_length_max": 881, + "prompt_length_std": 159.78, + "target_length_mean": 1 + }, + { + "name": "gaokao-history", + "sample_count": 235, + "prompt_length_mean": 141.48, + "prompt_length_min": 67, + "prompt_length_max": 314, + "prompt_length_std": 43.08, + "target_length_mean": 1 + }, + { + "name": "gaokao-biology", + "sample_count": 210, + "prompt_length_mean": 203.98, + "prompt_length_min": 75, + "prompt_length_max": 685, + "prompt_length_std": 93.18, + "target_length_mean": 1 + }, + { + "name": "gaokao-chemistry", + "sample_count": 207, + "prompt_length_mean": 348.37, + "prompt_length_min": 58, + "prompt_length_max": 1454, + "prompt_length_std": 246.96, + "target_length_mean": 1 + }, + { + "name": "gaokao-physics", + "sample_count": 200, + "prompt_length_mean": 251.9, + "prompt_length_min": 58, + "prompt_length_max": 581, + "prompt_length_std": 124.76, + "target_length_mean": 2.7 + }, + { + "name": "gaokao-mathqa", + "sample_count": 351, + "prompt_length_mean": 201.59, + "prompt_length_min": 93, + "prompt_length_max": 615, + "prompt_length_std": 83.9, + "target_length_mean": 1.04 + }, + { + "name": "jec-qa-kd", + "sample_count": 1000, + "prompt_length_mean": 170.43, + "prompt_length_min": 54, + "prompt_length_max": 454, + "prompt_length_std": 63.63, + "target_length_mean": 9.81 + }, + { + "name": "jec-qa-ca", + "sample_count": 1000, + "prompt_length_mean": 240.71, + "prompt_length_min": 79, + "prompt_length_max": 883, + "prompt_length_std": 88.98, + "target_length_mean": 8.98 + }, + { + "name": "math", + "sample_count": 1000, + "prompt_length_mean": 211.95, + "prompt_length_min": 40, + "prompt_length_max": 2186, + "prompt_length_std": 183.64, + "target_length_mean": 6.58 + }, + { + "name": "gaokao-mathcloze", + "sample_count": 118, + "prompt_length_mean": 123.42, + "prompt_length_min": 48, + "prompt_length_max": 501, + "prompt_length_std": 66.87, + "target_length_mean": 12.39 + } + ], + "prompt_length": { + "mean": 673.58, + "min": 40, + "max": 5316, + "std": 977.17 + }, + "target_length_mean": 3.91, + "computed_at": "2026-07-03T17:24:48.680869" + }, + "sample_example": { + "data": { + "input": [ + { + "id": "e28353e5", + "content": "Q: A car is being driven, in a straight line and at a uniform speed, towards the base of a vertical tower. The top of the tower is observed from the car and, in the process, it takes 10 minutes for the angle of elevation to change from 45° to 60°. After how much more time will this car reach the base of the tower? Answer Choices: (A)5(√3 + 1) (B)6(√3 + √2) (C)7(√3 – 1) (D)8(√3 – 2) (E)None of these\nA: Among A through E, the answer is" + } + ], + "choices": [ + "(A)5(√3 + 1)", + "(B)6(√3 + √2)", + "(C)7(√3 – 1)", + "(D)8(√3 – 2)", + "(E)None of these" + ], + "target": "A", + "id": 0, + "group_id": 0, + "metadata": { + "subset": "aqua-rat", + "has_passage": false + } + }, + "subset": "aqua-rat", + "truncated": false + }, + "readme": { + "en": "# AGIEval\n\n\n## Overview\n\nAGIEval is a human-centric benchmark designed to evaluate foundation models in the context of human cognition and problem-solving. It uses official, standard, and authoritative admission and qualification exams intended for general human test-takers, such as college entrance exams (GaoKao), law school admission tests (LSAT), math competitions, and lawyer qualification exams.\n\n## Task Description\n\n- **Task Type**: Mixed (Multiple-Choice QA + Open-ended Math)\n- **Input**: Questions from standardized exams with optional passages and answer choices\n- **Output**: Answer letter(s) for MCQ, or numerical/mathematical answer for open-ended\n- **Languages**: English and Chinese\n\n## Key Features\n\n- 21 subsets covering diverse exam types across two languages\n- English MCQ: LSAT (AR/LR/RC), SAT (Math/English), AQuA-RAT, LogiQA, GaoKao-English\n- Chinese MCQ: GaoKao (Chinese/Geography/History/Biology/Chemistry/Physics/MathQA), LogiQA-zh, JEC-QA\n- Open-ended math: MATH (English), GaoKao-MathCloze (Chinese)\n- Multi-select subsets: JEC-QA-KD, JEC-QA-CA, GaoKao-Physics\n- Includes passage-based reading comprehension questions\n\n## Evaluation Notes\n\n- MCQ subsets use evalscope's standard MultiChoice template and extraction\n- Multi-select subsets use Chinese multi-answer template\n- Math/cloze subsets use mathematical equivalence checking\n- CoT (Chain-of-Thought) prompting enabled by default\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `agieval` |\n| **Dataset ID** | [opencompass/agieval](https://modelscope.cn/datasets/opencompass/agieval/summary) |\n| **Paper** | N/A |\n| **Tags** | `Knowledge`, `MCQ`, `Math`, `Reasoning` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n| **Train Split** | `dev` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 8,269 |\n| Prompt Length (Mean) | 673.58 chars |\n| Prompt Length (Min/Max) | 40 / 5316 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `aqua-rat` | 254 | 290.09 | 103 | 587 |\n| `logiqa-en` | 651 | 911.89 | 248 | 1769 |\n| `lsat-ar` | 230 | 946.36 | 635 | 1853 |\n| `lsat-lr` | 510 | 1156.66 | 563 | 2348 |\n| `lsat-rc` | 269 | 3652.86 | 2959 | 4825 |\n| `sat-math` | 220 | 392.45 | 120 | 1201 |\n| `sat-en` | 206 | 4618.28 | 3569 | 5316 |\n| `sat-en-without-passage` | 206 | 435.91 | 169 | 937 |\n| `gaokao-english` | 306 | 2025.44 | 517 | 4216 |\n| `logiqa-zh` | 651 | 267.62 | 98 | 526 |\n| `gaokao-chinese` | 246 | 988.09 | 152 | 2186 |\n| `gaokao-geography` | 199 | 204.82 | 64 | 881 |\n| `gaokao-history` | 235 | 141.48 | 67 | 314 |\n| `gaokao-biology` | 210 | 203.98 | 75 | 685 |\n| `gaokao-chemistry` | 207 | 348.37 | 58 | 1454 |\n| `gaokao-physics` | 200 | 251.9 | 58 | 581 |\n| `gaokao-mathqa` | 351 | 201.59 | 93 | 615 |\n| `jec-qa-kd` | 1,000 | 170.43 | 54 | 454 |\n| `jec-qa-ca` | 1,000 | 240.71 | 79 | 883 |\n| `math` | 1,000 | 211.95 | 40 | 2186 |\n| `gaokao-mathcloze` | 118 | 123.42 | 48 | 501 |\n\n## Sample Example\n\n**Subset**: `aqua-rat`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"e28353e5\",\n \"content\": \"Q: A car is being driven, in a straight line and at a uniform speed, towards the base of a vertical tower. The top of the tower is observed from the car and, in the process, it takes 10 minutes for the angle of elevation to change from 45° to 60°. After how much more time will this car reach the base of the tower? Answer Choices: (A)5(√3 + 1) (B)6(√3 + √2) (C)7(√3 – 1) (D)8(√3 – 2) (E)None of these\\nA: Among A through E, the answer is\"\n }\n ],\n \"choices\": [\n \"(A)5(√3 + 1)\",\n \"(B)6(√3 + √2)\",\n \"(C)7(√3 – 1)\",\n \"(D)8(√3 – 2)\",\n \"(E)None of these\"\n ],\n \"target\": \"A\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"subset\": \"aqua-rat\",\n \"has_passage\": false\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\nAnswer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of {letters}. Think step by step before answering.\n\n{question}\n\n{choices}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets agieval \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['agieval'],\n dataset_args={\n 'agieval': {\n # subset_list: ['aqua-rat', 'logiqa-en', 'lsat-ar'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# AGIEval\n\n\n## 概述\n\nAGIEval 是一个以人类为中心的基准测试，旨在评估基础模型在人类认知与问题解决场景下的能力。该基准采用面向普通人类考生的官方、标准且权威的入学及资格考试题目，例如高考（GaoKao）、法学院入学考试（LSAT）、数学竞赛以及律师资格考试等。\n\n## 任务描述\n\n- **任务类型**：混合型（多项选择问答 + 开放式数学题）\n- **输入**：标准化考试中的题目，可包含段落和选项\n- **输出**：多项选择题的答案字母，或开放式题目的数值/数学答案\n- **语言**：英语和中文\n\n## 主要特点\n\n- 包含21个子集，涵盖两种语言下的多种考试类型\n- 英语多项选择题：LSAT（AR/LR/RC）、SAT（数学/英语）、AQuA-RAT、LogiQA、GaoKao-English\n- 中文多项选择题：GaoKao（语文/地理/历史/生物/化学/物理/MathQA）、LogiQA-zh、JEC-QA\n- 开放式数学题：MATH（英语）、GaoKao-MathCloze（中文）\n- 多选子集：JEC-QA-KD、JEC-QA-CA、GaoKao-Physics\n- 包含基于段落的阅读理解题目\n\n## 评估说明\n\n- 多项选择题子集使用 evalscope 的标准 MultiChoice 模板和答案提取方法\n- 多选子集使用中文多答案模板\n- 数学/填空类子集采用数学等价性检查\n- 默认启用 CoT（思维链）提示\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `agieval` |\n| **数据集ID** | [opencompass/agieval](https://modelscope.cn/datasets/opencompass/agieval/summary) |\n| **论文** | N/A |\n| **标签** | `Knowledge`, `MCQ`, `Math`, `Reasoning` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n| **训练划分** | `dev` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 8,269 |\n| 提示词长度（平均） | 673.58 字符 |\n| 提示词长度（最小/最大） | 40 / 5316 字符 |\n\n**各子集统计数据：**\n\n| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 |\n|--------|---------|-------------|------------|------------|\n| `aqua-rat` | 254 | 290.09 | 103 | 587 |\n| `logiqa-en` | 651 | 911.89 | 248 | 1769 |\n| `lsat-ar` | 230 | 946.36 | 635 | 1853 |\n| `lsat-lr` | 510 | 1156.66 | 563 | 2348 |\n| `lsat-rc` | 269 | 3652.86 | 2959 | 4825 |\n| `sat-math` | 220 | 392.45 | 120 | 1201 |\n| `sat-en` | 206 | 4618.28 | 3569 | 5316 |\n| `sat-en-without-passage` | 206 | 435.91 | 169 | 937 |\n| `gaokao-english` | 306 | 2025.44 | 517 | 4216 |\n| `logiqa-zh` | 651 | 267.62 | 98 | 526 |\n| `gaokao-chinese` | 246 | 988.09 | 152 | 2186 |\n| `gaokao-geography` | 199 | 204.82 | 64 | 881 |\n| `gaokao-history` | 235 | 141.48 | 67 | 314 |\n| `gaokao-biology` | 210 | 203.98 | 75 | 685 |\n| `gaokao-chemistry` | 207 | 348.37 | 58 | 1454 |\n| `gaokao-physics` | 200 | 251.9 | 58 | 581 |\n| `gaokao-mathqa` | 351 | 201.59 | 93 | 615 |\n| `jec-qa-kd` | 1,000 | 170.43 | 54 | 454 |\n| `jec-qa-ca` | 1,000 | 240.71 | 79 | 883 |\n| `math` | 1,000 | 211.95 | 40 | 2186 |\n| `gaokao-mathcloze` | 118 | 123.42 | 48 | 501 |\n\n## 样例示例\n\n**子集**: `aqua-rat`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"e28353e5\",\n \"content\": \"Q: A car is being driven, in a straight line and at a uniform speed, towards the base of a vertical tower. The top of the tower is observed from the car and, in the process, it takes 10 minutes for the angle of elevation to change from 45° to 60°. After how much more time will this car reach the base of the tower? Answer Choices: (A)5(√3 + 1) (B)6(√3 + √2) (C)7(√3 – 1) (D)8(√3 – 2) (E)None of these\\nA: Among A through E, the answer is\"\n }\n ],\n \"choices\": [\n \"(A)5(√3 + 1)\",\n \"(B)6(√3 + √2)\",\n \"(C)7(√3 – 1)\",\n \"(D)8(√3 – 2)\",\n \"(E)None of these\"\n ],\n \"target\": \"A\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"subset\": \"aqua-rat\",\n \"has_passage\": false\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n请回答以下多项选择题。你的回答最后一行应为以下格式：'ANSWER: [LETTER]'（不含引号），其中 [LETTER] 是 {letters} 中的一个。请逐步思考后再作答。\n\n{question}\n\n{choices}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets agieval \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['agieval'],\n dataset_args={\n 'agieval': {\n # subset_list: ['aqua-rat', 'logiqa-en', 'lsat-ar'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "08864ede0d35c3544f941135fe6937ae", + "needs_translation": false + }, + "updated_at": "2026-07-06T14:05:00.151489", + "translation_updated_at": "2026-07-06T14:05:25" +} diff --git a/evalscope/evalscope/benchmarks/_meta/ai2d.json b/evalscope/evalscope/benchmarks/_meta/ai2d.json index 0e580b7..3668d5c 100644 --- a/evalscope/evalscope/benchmarks/_meta/ai2d.json +++ b/evalscope/evalscope/benchmarks/_meta/ai2d.json @@ -148,4 +148,4 @@ }, "updated_at": "2026-01-28T17:31:32.180631", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/aime24.json b/evalscope/evalscope/benchmarks/_meta/aime24.json index c2e82fc..5de8acd 100644 --- a/evalscope/evalscope/benchmarks/_meta/aime24.json +++ b/evalscope/evalscope/benchmarks/_meta/aime24.json @@ -74,4 +74,4 @@ }, "updated_at": "2026-03-16T17:43:27.479633", "translation_updated_at": "2026-03-16T17:46:34Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/aime25.json b/evalscope/evalscope/benchmarks/_meta/aime25.json index 1bd43b5..4003e01 100644 --- a/evalscope/evalscope/benchmarks/_meta/aime25.json +++ b/evalscope/evalscope/benchmarks/_meta/aime25.json @@ -74,4 +74,4 @@ }, "updated_at": "2026-03-16T17:43:27.477945", "translation_updated_at": "2026-03-16T17:46:34Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/aime26.json b/evalscope/evalscope/benchmarks/_meta/aime26.json index 0f5403b..c030d8c 100644 --- a/evalscope/evalscope/benchmarks/_meta/aime26.json +++ b/evalscope/evalscope/benchmarks/_meta/aime26.json @@ -74,4 +74,4 @@ }, "updated_at": "2026-03-16T17:43:27.476270", "translation_updated_at": "2026-03-16T17:46:34Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/air_bench_chat.json b/evalscope/evalscope/benchmarks/_meta/air_bench_chat.json index f3a47bb..1a97490 100644 --- a/evalscope/evalscope/benchmarks/_meta/air_bench_chat.json +++ b/evalscope/evalscope/benchmarks/_meta/air_bench_chat.json @@ -334,4 +334,4 @@ }, "updated_at": "2026-05-18T10:23:16.641619", "translation_updated_at": "2026-05-18T10:23:22" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/air_bench_foundation.json b/evalscope/evalscope/benchmarks/_meta/air_bench_foundation.json index a8f4d3e..14db2d8 100644 --- a/evalscope/evalscope/benchmarks/_meta/air_bench_foundation.json +++ b/evalscope/evalscope/benchmarks/_meta/air_bench_foundation.json @@ -808,4 +808,4 @@ }, "updated_at": "2026-05-18T10:23:16.650501", "translation_updated_at": "2026-05-18T10:23:22" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/alpaca_eval.json b/evalscope/evalscope/benchmarks/_meta/alpaca_eval.json index 65a05dd..40048b2 100644 --- a/evalscope/evalscope/benchmarks/_meta/alpaca_eval.json +++ b/evalscope/evalscope/benchmarks/_meta/alpaca_eval.json @@ -74,4 +74,4 @@ }, "updated_at": "2026-01-28T17:31:32.181393", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/amc.json b/evalscope/evalscope/benchmarks/_meta/amc.json index b401f9a..72c8308 100644 --- a/evalscope/evalscope/benchmarks/_meta/amc.json +++ b/evalscope/evalscope/benchmarks/_meta/amc.json @@ -99,4 +99,4 @@ }, "updated_at": "2026-01-28T17:31:32.181373", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/anat_em.json b/evalscope/evalscope/benchmarks/_meta/anat_em.json index 05ec4ae..396ee51 100644 --- a/evalscope/evalscope/benchmarks/_meta/anat_em.json +++ b/evalscope/evalscope/benchmarks/_meta/anat_em.json @@ -99,4 +99,4 @@ }, "updated_at": "2026-01-28T17:31:32.334291", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/arc.json b/evalscope/evalscope/benchmarks/_meta/arc.json index ae285ba..bfedc3d 100644 --- a/evalscope/evalscope/benchmarks/_meta/arc.json +++ b/evalscope/evalscope/benchmarks/_meta/arc.json @@ -89,4 +89,4 @@ }, "updated_at": "2026-01-28T17:31:32.181351", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/arc_agi_2.json b/evalscope/evalscope/benchmarks/_meta/arc_agi_2.json new file mode 100644 index 0000000..953af87 --- /dev/null +++ b/evalscope/evalscope/benchmarks/_meta/arc_agi_2.json @@ -0,0 +1,76 @@ +{ + "meta": { + "pretty_name": "ARC-AGI-2", + "dataset_id": "evalscope/arc-agi-2", + "paper_url": null, + "tags": [ + "Reasoning" + ], + "metrics": [ + "acc" + ], + "few_shot_num": 0, + "eval_split": "test", + "train_split": "", + "subset_list": [ + "default" + ], + "description": "\n## Overview\n\nARC-AGI-2 (Abstraction and Reasoning Corpus for Artificial General Intelligence 2) is a benchmark designed to measure an AI system's ability to efficiently acquire new skills on-the-fly, using only a handful of demonstrations. It evaluates abstract reasoning and pattern recognition through grid transformation tasks.\n\n## Task Description\n\n- **Task Type**: Abstract Reasoning / Pattern Recognition\n- **Input**: A series of input-output grid pairs (demonstrations) followed by a test input grid\n- **Output**: The predicted output grid matching the inferred transformation rule\n- **Grid Format**: 2D arrays of integers (0-9), variable sizes (up to 30x30)\n\n## Key Features\n\n- 1,000 public training tasks and 120 public evaluation tasks\n- Each task provides 2-10 demonstration input/output pairs\n- Models must infer the transformation rule from demonstrations\n- Tests abstract reasoning without reliance on learned knowledge\n- Pixel-perfect output required (exact grid match)\n\n## Evaluation Notes\n\n- Scoring is based on **exact grid match** (shape and all values must be identical)\n- Models must output the grid as a JSON 2D array\n- Zero-shot evaluation (demonstrations are provided within each task)\n- Designed to be solvable by humans but challenging for AI\n", + "prompt_template": "{question}", + "system_prompt": "You are an expert at abstract reasoning and pattern recognition. Given input-output grid pairs as examples, you must figure out the transformation rule and apply it to a new test input to produce the correct output grid.", + "few_shot_prompt_template": "", + "aggregation": "mean_and_pass_hat_k", + "extra_params": {}, + "sandbox_config": {}, + "category": "llm" + }, + "statistics": { + "total_samples": 120, + "subset_stats": [ + { + "name": "default", + "sample_count": 120, + "prompt_length_mean": 8026.79, + "prompt_length_min": 2437, + "prompt_length_max": 25471, + "prompt_length_std": 4038.76, + "target_length_mean": 1328.32 + } + ], + "prompt_length": { + "mean": 8026.79, + "min": 2437, + "max": 25471, + "std": 4038.76 + }, + "target_length_mean": 1328.32, + "computed_at": "2026-07-03T17:22:52.935931" + }, + "sample_example": { + "data": { + "input": [ + { + "id": "7b97143f", + "content": "You are an expert at abstract reasoning and pattern recognition. Given input-output grid pairs as examples, you must figure out the transformation rule and apply it to a new test input to produce the correct output grid." + }, + { + "id": "bdaa2b22", + "content": "You are given a series of input-output grid pairs as examples. Each grid is a 2D array of integers (0-9). Study the pattern in the examples, then predict the output for the test input.\n\nExamples:\nExample 1:\nInput: [[0, 0, 0, 0, 0, 0, 0, 0, 0, ... [TRUNCATED 7482 chars] ... 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]\n\nProvide the output grid as a JSON 2D array. Only output the JSON array, nothing else." + } + ], + "target": "[[8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 8, 8, 8], [8, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0], [8, 0, 8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 8, 8, 8, 8, 0], [ ... [TRUNCATED 1596 chars] ... ], [8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0, 0, 0, 8, 0], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 0, 8, 8, 8, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0, 0, 0]]", + "id": 0, + "group_id": 0 + }, + "subset": "default", + "truncated": false + }, + "readme": { + "en": "# ARC-AGI-2\n\n\n## Overview\n\nARC-AGI-2 (Abstraction and Reasoning Corpus for Artificial General Intelligence 2) is a benchmark designed to measure an AI system's ability to efficiently acquire new skills on-the-fly, using only a handful of demonstrations. It evaluates abstract reasoning and pattern recognition through grid transformation tasks.\n\n## Task Description\n\n- **Task Type**: Abstract Reasoning / Pattern Recognition\n- **Input**: A series of input-output grid pairs (demonstrations) followed by a test input grid\n- **Output**: The predicted output grid matching the inferred transformation rule\n- **Grid Format**: 2D arrays of integers (0-9), variable sizes (up to 30x30)\n\n## Key Features\n\n- 1,000 public training tasks and 120 public evaluation tasks\n- Each task provides 2-10 demonstration input/output pairs\n- Models must infer the transformation rule from demonstrations\n- Tests abstract reasoning without reliance on learned knowledge\n- Pixel-perfect output required (exact grid match)\n\n## Evaluation Notes\n\n- Scoring is based on **exact grid match** (shape and all values must be identical)\n- Models must output the grid as a JSON 2D array\n- Zero-shot evaluation (demonstrations are provided within each task)\n- Designed to be solvable by humans but challenging for AI\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `arc_agi_2` |\n| **Dataset ID** | [evalscope/arc-agi-2](https://modelscope.cn/datasets/evalscope/arc-agi-2/summary) |\n| **Paper** | N/A |\n| **Tags** | `Reasoning` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n| **Aggregation** | `mean_and_pass_hat_k` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 120 |\n| Prompt Length (Mean) | 8026.79 chars |\n| Prompt Length (Min/Max) | 2437 / 25471 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"7b97143f\",\n \"content\": \"You are an expert at abstract reasoning and pattern recognition. Given input-output grid pairs as examples, you must figure out the transformation rule and apply it to a new test input to produce the correct output grid.\"\n },\n {\n \"id\": \"bdaa2b22\",\n \"content\": \"You are given a series of input-output grid pairs as examples. Each grid is a 2D array of integers (0-9). Study the pattern in the examples, then predict the output for the test input.\\n\\nExamples:\\nExample 1:\\nInput: [[0, 0, 0, 0, 0, 0, 0, 0, 0, ... [TRUNCATED 7482 chars] ... 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]\\n\\nProvide the output grid as a JSON 2D array. Only output the JSON array, nothing else.\"\n }\n ],\n \"target\": \"[[8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 8, 8, 8], [8, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0], [8, 0, 8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 8, 8, 8, 8, 0], [ ... [TRUNCATED 1596 chars] ... ], [8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0, 0, 0, 8, 0], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 0, 8, 8, 8, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0, 0, 0]]\",\n \"id\": 0,\n \"group_id\": 0\n}\n```\n\n## Prompt Template\n\n**System Prompt:**\n```text\nYou are an expert at abstract reasoning and pattern recognition. Given input-output grid pairs as examples, you must figure out the transformation rule and apply it to a new test input to produce the correct output grid.\n```\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets arc_agi_2 \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['arc_agi_2'],\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# ARC-AGI-2\n\n\n## 概述\n\nARC-AGI-2（面向人工通用智能的抽象与推理语料库 2）是一个旨在衡量 AI 系统能否仅凭少量示例就高效地即时掌握新技能的基准测试。它通过网格变换任务来评估模型的抽象推理和模式识别能力。\n\n## 任务描述\n\n- **任务类型**：抽象推理 / 模式识别\n- **输入**：一系列输入-输出网格对（示例），后跟一个测试输入网格\n- **输出**：根据推断出的变换规则生成的预测输出网格\n- **网格格式**：二维整数数组（0-9），尺寸可变（最大 30x30）\n\n## 主要特点\n\n- 包含 1,000 个公开训练任务和 120 个公开评估任务\n- 每个任务提供 2-10 个示例输入/输出对\n- 模型必须从示例中推断出变换规则\n- 测试抽象推理能力，不依赖于已学习的知识\n- 要求输出像素级精确（必须完全匹配目标网格）\n\n## 评估说明\n\n- 评分基于**精确网格匹配**（形状和所有数值必须完全一致）\n- 模型必须以 JSON 二维数组格式输出网格\n- 零样本评估（每个任务内提供示例）\n- 设计为人类可解但对 AI 具有挑战性\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `arc_agi_2` |\n| **数据集ID** | [evalscope/arc-agi-2](https://modelscope.cn/datasets/evalscope/arc-agi-2/summary) |\n| **论文** | N/A |\n| **标签** | `Reasoning` |\n| **指标** | `acc` |\n| **默认样本数** | 0-shot |\n| **评估划分** | `test` |\n| **聚合方式** | `mean_and_pass_hat_k` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 120 |\n| 提示词长度（平均） | 8026.79 字符 |\n| 提示词长度（最小/最大） | 2437 / 25471 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"7b97143f\",\n \"content\": \"You are an expert at abstract reasoning and pattern recognition. Given input-output grid pairs as examples, you must figure out the transformation rule and apply it to a new test input to produce the correct output grid.\"\n },\n {\n \"id\": \"bdaa2b22\",\n \"content\": \"You are given a series of input-output grid pairs as examples. Each grid is a 2D array of integers (0-9). Study the pattern in the examples, then predict the output for the test input.\\n\\nExamples:\\nExample 1:\\nInput: [[0, 0, 0, 0, 0, 0, 0, 0, 0, ... [TRUNCATED 7482 chars] ... 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]\\n\\nProvide the output grid as a JSON 2D array. Only output the JSON array, nothing else.\"\n }\n ],\n \"target\": \"[[8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 8, 8, 8], [8, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0], [8, 0, 8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 8, 8, 8, 8, 0], [ ... [TRUNCATED 1596 chars] ... ], [8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0, 0, 0, 8, 0], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 0, 8, 8, 8, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0, 0, 0]]\",\n \"id\": 0,\n \"group_id\": 0\n}\n```\n\n## 提示模板\n\n**系统提示**：\n```text\nYou are an expert at abstract reasoning and pattern recognition. Given input-output grid pairs as examples, you must figure out the transformation rule and apply it to a new test input to produce the correct output grid.\n```\n\n**提示模板**：\n```text\n{question}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets arc_agi_2 \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['arc_agi_2'],\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "5946771be95fe7121f09dff910dd6da0", + "needs_translation": false + }, + "updated_at": "2026-07-06T14:05:00.208824", + "translation_updated_at": "2026-07-06T14:05:25" +} diff --git a/evalscope/evalscope/benchmarks/_meta/arena_hard.json b/evalscope/evalscope/benchmarks/_meta/arena_hard.json index 20a4ee5..3c358f5 100644 --- a/evalscope/evalscope/benchmarks/_meta/arena_hard.json +++ b/evalscope/evalscope/benchmarks/_meta/arena_hard.json @@ -73,4 +73,4 @@ }, "updated_at": "2026-01-28T17:31:32.190759", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/arxivmath.json b/evalscope/evalscope/benchmarks/_meta/arxivmath.json index 2eb65e7..4fb682f 100644 --- a/evalscope/evalscope/benchmarks/_meta/arxivmath.json +++ b/evalscope/evalscope/benchmarks/_meta/arxivmath.json @@ -112,4 +112,4 @@ }, "updated_at": "2026-07-03T16:31:10.439972", "translation_updated_at": "2026-07-03T16:31:16" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/arxivrollbench.json b/evalscope/evalscope/benchmarks/_meta/arxivrollbench.json index c794810..078c420 100644 --- a/evalscope/evalscope/benchmarks/_meta/arxivrollbench.json +++ b/evalscope/evalscope/benchmarks/_meta/arxivrollbench.json @@ -792,4 +792,4 @@ }, "updated_at": "2026-05-26T16:07:01.880371", "translation_updated_at": "2026-05-26T16:07:57" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/arxivrollbench_full.json b/evalscope/evalscope/benchmarks/_meta/arxivrollbench_full.json index b9592dc..6d6c7cc 100644 --- a/evalscope/evalscope/benchmarks/_meta/arxivrollbench_full.json +++ b/evalscope/evalscope/benchmarks/_meta/arxivrollbench_full.json @@ -792,4 +792,4 @@ }, "updated_at": "2026-05-26T16:16:11.135223", "translation_updated_at": "2026-05-26T16:18:54" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/baby_vision.json b/evalscope/evalscope/benchmarks/_meta/baby_vision.json index 76f4097..b309068 100644 --- a/evalscope/evalscope/benchmarks/_meta/baby_vision.json +++ b/evalscope/evalscope/benchmarks/_meta/baby_vision.json @@ -283,4 +283,4 @@ }, "updated_at": "2026-07-03T11:09:47.265495", "translation_updated_at": "2026-07-03T16:19:29" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/bbh.json b/evalscope/evalscope/benchmarks/_meta/bbh.json index 1e1cfb4..e401056 100644 --- a/evalscope/evalscope/benchmarks/_meta/bbh.json +++ b/evalscope/evalscope/benchmarks/_meta/bbh.json @@ -333,4 +333,4 @@ }, "updated_at": "2026-01-28T17:31:32.191108", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/bc2gm.json b/evalscope/evalscope/benchmarks/_meta/bc2gm.json index 5a01e83..282e8ab 100644 --- a/evalscope/evalscope/benchmarks/_meta/bc2gm.json +++ b/evalscope/evalscope/benchmarks/_meta/bc2gm.json @@ -123,4 +123,4 @@ }, "updated_at": "2026-01-28T17:31:32.338567", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/bc4chemd.json b/evalscope/evalscope/benchmarks/_meta/bc4chemd.json index 96b1765..6908888 100644 --- a/evalscope/evalscope/benchmarks/_meta/bc4chemd.json +++ b/evalscope/evalscope/benchmarks/_meta/bc4chemd.json @@ -135,4 +135,4 @@ }, "updated_at": "2026-01-28T17:31:32.339790", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/bc5cdr.json b/evalscope/evalscope/benchmarks/_meta/bc5cdr.json index 202d521..412425f 100644 --- a/evalscope/evalscope/benchmarks/_meta/bc5cdr.json +++ b/evalscope/evalscope/benchmarks/_meta/bc5cdr.json @@ -123,4 +123,4 @@ }, "updated_at": "2026-01-28T17:31:32.341754", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/bfcl_v3.json b/evalscope/evalscope/benchmarks/_meta/bfcl_v3.json index 205ceab..c0c390e 100644 --- a/evalscope/evalscope/benchmarks/_meta/bfcl_v3.json +++ b/evalscope/evalscope/benchmarks/_meta/bfcl_v3.json @@ -336,4 +336,4 @@ }, "updated_at": "2026-01-28T17:31:32.200319", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/bfcl_v4.json b/evalscope/evalscope/benchmarks/_meta/bfcl_v4.json index f3e97ad..0d899c9 100644 --- a/evalscope/evalscope/benchmarks/_meta/bfcl_v4.json +++ b/evalscope/evalscope/benchmarks/_meta/bfcl_v4.json @@ -332,4 +332,4 @@ }, "updated_at": "2026-01-28T17:31:32.199735", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/bigcodebench.json b/evalscope/evalscope/benchmarks/_meta/bigcodebench.json index 5e25e5c..7ef8549 100644 --- a/evalscope/evalscope/benchmarks/_meta/bigcodebench.json +++ b/evalscope/evalscope/benchmarks/_meta/bigcodebench.json @@ -39,6 +39,21 @@ "type": "bool", "description": "Whether to prepend code_prompt to the solution for function signature alignment.", "value": true + }, + "docker_build_context": { + "type": "str", + "description": "Optional local Docker build context. When set, overrides the default sandbox image.", + "value": "" + }, + "dockerfile": { + "type": "str", + "description": "Dockerfile path inside docker_build_context.", + "value": "Dockerfile" + }, + "force_rebuild": { + "type": "bool", + "description": "Force rebuilding the optional local Docker image.", + "value": false } }, "sandbox_config": { @@ -86,11 +101,11 @@ "truncated": false }, "readme": { - "en": "# BigCodeBench\n\n\n## Overview\n\nBigCodeBench is an easy-to-use benchmark for solving practical and challenging tasks via code. It evaluates the true programming capabilities of large language models (LLMs) in a more realistic setting with diverse function calls from 139 popular libraries covering 723 API calls.\n\n## Task Description\n\n- **Task Type**: Code Generation (Python)\n- **Input**: Programming task description (docstring or natural language instruction)\n- **Output**: Complete Python function implementation\n- **Libraries**: 139 popular Python libraries (numpy, pandas, sklearn, etc.)\n\n## Key Features\n\n- 1,140 rich-context programming tasks in Python\n- Two evaluation modes: Complete (docstring) and Instruct (natural language)\n- Covers diverse function calls from 139 popular libraries\n- Uses unittest.TestCase for thorough correctness verification\n- Supports pass@k metric calculation\n\n## Evaluation Notes\n\n- **Sandbox Required**: Requires sandbox environment with 70+ Python libraries pre-installed\n- Two modes available via `split` parameter: `complete` (docstring completion) or `instruct` (NL instruction)\n- Default timeout is 240 seconds per problem\n- `calibrate` option prepends code_prompt to align function signatures\n- See [sandbox documentation](https://evalscope.readthedocs.io/en/latest/user_guides/sandbox.html) for setup\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `bigcodebench` |\n| **Dataset ID** | [evalscope/bigcodebench](https://modelscope.cn/datasets/evalscope/bigcodebench/summary) |\n| **Paper** | N/A |\n| **Tags** | `Coding` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `v0.1.4` |\n| **Aggregation** | `mean_and_pass_at_k` |\n\n\n## Data Statistics\n\n*Statistics not available.*\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"657d2473\",\n \"content\": \"Calculates the average of the sums of absolute differences between each pair of consecutive numbers for all permutations of a given list. Each permutation is shuffled before calculating the differences. Args: - numbers (list): A list of numbe ... [TRUNCATED 75 chars] ... loat: The average of the sums of absolute differences for each shuffled permutation of the list.\\nYou should write self-contained code starting with:\\n```\\nimport itertools\\nfrom random import shuffle\\ndef task_func(numbers=list(range(1, 3))):\\n```\"\n }\n ],\n \"target\": \" permutations = list(itertools.permutations(numbers))\\n sum_diffs = 0\\n\\n for perm in permutations:\\n perm = list(perm)\\n shuffle(perm)\\n diffs = [abs(perm[i] - perm[i+1]) for i in range(len(perm)-1)]\\n sum_diffs += sum(diffs)\\n\\n avg_sum_diffs = sum_diffs / len(permutations)\\n \\n return avg_sum_diffs\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"task_id\": \"BigCodeBench/0\",\n \"entry_point\": \"task_func\",\n \"complete_prompt\": \"import itertools\\nfrom random import shuffle\\n\\ndef task_func(numbers=list(range(1, 3))):\\n \\\"\\\"\\\"\\n Calculates the average of the sums of absolute differences between each pair of consecutive numbers \\n for all permutations of a given list. ... [TRUNCATED 187 chars] ... age of the sums of absolute differences for each shuffled permutation of the list.\\n\\n Requirements:\\n - itertools\\n - random.shuffle\\n\\n Example:\\n >>> result = task_func([1, 2, 3])\\n >>> isinstance(result, float)\\n True\\n \\\"\\\"\\\"\\n\",\n \"code_prompt\": \"import itertools\\nfrom random import shuffle\\ndef task_func(numbers=list(range(1, 3))):\\n\",\n \"test\": \"import unittest\\nfrom unittest.mock import patch\\nfrom random import seed, shuffle\\nimport itertools\\nclass TestCases(unittest.TestCase):\\n def test_default_numbers(self):\\n # Test with default number range (1 to 10) to check that the res ... [TRUNCATED 2578 chars] ... x: seed(1) or shuffle(x)):\\n result1 = task_func([1, 2, 3])\\n with patch('random.shuffle', side_effect=lambda x: seed(1) or shuffle(x)):\\n result2 = task_func([1, 2, 4])\\n self.assertNotEqual(result1, result2)\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{prompt}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `split` | `str` | `instruct` | Evaluation mode: \"complete\" (docstring completion) or \"instruct\" (NL instruction). Choices: ['complete', 'instruct'] |\n| `version` | `str` | `default` | Dataset version. Use \"default\" for the latest available version. |\n| `calibrate` | `bool` | `True` | Whether to prepend code_prompt to the solution for function signature alignment. |\n\n## Sandbox Configuration\n\nThis benchmark requires a sandbox environment for code execution.\n\n```json\n{\n \"image\": \"bigcodebench/bigcodebench-evaluate:latest\",\n \"tools_config\": {\n \"shell_executor\": {},\n \"python_executor\": {}\n },\n \"memory_limit\": \"4g\"\n}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets bigcodebench \\\n --sandbox '{\"enabled\": true}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['bigcodebench'],\n sandbox={'enabled': True},\n dataset_args={\n 'bigcodebench': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# BigCodeBench\n\n\n## 概述\n\nBigCodeBench 是一个易于使用的基准测试，用于通过代码解决实际且具有挑战性的任务。它在更贴近现实的场景中评估大语言模型（LLMs）的真实编程能力，涵盖来自 139 个流行库的 723 个 API 调用。\n\n## 任务描述\n\n- **任务类型**：代码生成（Python）\n- **输入**：编程任务描述（文档字符串或自然语言指令）\n- **输出**：完整的 Python 函数实现\n- **涉及库**：139 个流行的 Python 库（如 numpy、pandas、sklearn 等）\n\n## 主要特性\n\n- 包含 1,140 个上下文丰富的 Python 编程任务\n- 提供两种评估模式：Complete（基于文档字符串）和 Instruct（基于自然语言指令）\n- 覆盖来自 139 个流行库的多样化函数调用\n- 使用 `unittest.TestCase` 进行全面的正确性验证\n- 支持 pass@k 指标计算\n\n## 评估说明\n\n- **需要沙箱环境**：要求预装 70+ 个 Python 库的沙箱环境\n- 通过 `split` 参数选择两种模式：`complete`（文档字符串补全）或 `instruct`（自然语言指令）\n- 每个问题默认超时时间为 240 秒\n- 启用 `calibrate` 选项会在生成代码前添加 `code_prompt`，以对齐函数签名\n- 沙箱环境设置详见 [沙箱文档](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/sandbox.html)\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `bigcodebench` |\n| **数据集ID** | [evalscope/bigcodebench](https://modelscope.cn/datasets/evalscope/bigcodebench/summary) |\n| **论文** | N/A |\n| **标签** | `Coding` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分版本** | `v0.1.4` |\n| **聚合方式** | `mean_and_pass_at_k` |\n\n\n## 数据统计\n\n*统计数据暂不可用。*\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"657d2473\",\n \"content\": \"Calculates the average of the sums of absolute differences between each pair of consecutive numbers for all permutations of a given list. Each permutation is shuffled before calculating the differences. Args: - numbers (list): A list of numbe ... [TRUNCATED 75 chars] ... loat: The average of the sums of absolute differences for each shuffled permutation of the list.\\nYou should write self-contained code starting with:\\n```\\nimport itertools\\nfrom random import shuffle\\ndef task_func(numbers=list(range(1, 3))):\\n```\"\n }\n ],\n \"target\": \" permutations = list(itertools.permutations(numbers))\\n sum_diffs = 0\\n\\n for perm in permutations:\\n perm = list(perm)\\n shuffle(perm)\\n diffs = [abs(perm[i] - perm[i+1]) for i in range(len(perm)-1)]\\n sum_diffs += sum(diffs)\\n\\n avg_sum_diffs = sum_diffs / len(permutations)\\n \\n return avg_sum_diffs\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"task_id\": \"BigCodeBench/0\",\n \"entry_point\": \"task_func\",\n \"complete_prompt\": \"import itertools\\nfrom random import shuffle\\n\\ndef task_func(numbers=list(range(1, 3))):\\n \\\"\\\"\\\"\\n Calculates the average of the sums of absolute differences between each pair of consecutive numbers \\n for all permutations of a given list. ... [TRUNCATED 187 chars] ... age of the sums of absolute differences for each shuffled permutation of the list.\\n\\n Requirements:\\n - itertools\\n - random.shuffle\\n\\n Example:\\n >>> result = task_func([1, 2, 3])\\n >>> isinstance(result, float)\\n True\\n \\\"\\\"\\\"\\n\",\n \"code_prompt\": \"import itertools\\nfrom random import shuffle\\ndef task_func(numbers=list(range(1, 3))):\\n\",\n \"test\": \"import unittest\\nfrom unittest.mock import patch\\nfrom random import seed, shuffle\\nimport itertools\\nclass TestCases(unittest.TestCase):\\n def test_default_numbers(self):\\n # Test with default number range (1 to 10) to check that the res ... [TRUNCATED 2578 chars] ... x: seed(1) or shuffle(x)):\\n result1 = task_func([1, 2, 3])\\n with patch('random.shuffle', side_effect=lambda x: seed(1) or shuffle(x)):\\n result2 = task_func([1, 2, 4])\\n self.assertNotEqual(result1, result2)\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{prompt}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `split` | `str` | `instruct` | 评估模式：\"complete\"（文档字符串补全）或 \"instruct\"（自然语言指令）。可选值：['complete', 'instruct'] |\n| `version` | `str` | `default` | 数据集版本。使用 \"default\" 表示最新可用版本。 |\n| `calibrate` | `bool` | `True` | 是否在解决方案前添加 `code_prompt` 以对齐函数签名。 |\n\n## 沙箱配置\n\n此基准测试需要沙箱环境来执行代码。\n\n```json\n{\n \"image\": \"bigcodebench/bigcodebench-evaluate:latest\",\n \"tools_config\": {\n \"shell_executor\": {},\n \"python_executor\": {}\n },\n \"memory_limit\": \"4g\"\n}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets bigcodebench \\\n --sandbox '{\"enabled\": true}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['bigcodebench'],\n sandbox={'enabled': True},\n dataset_args={\n 'bigcodebench': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "86a2b4701845ac156c978de01a8aa00d", + "en": "# BigCodeBench\n\n\n## Overview\n\nBigCodeBench is an easy-to-use benchmark for solving practical and challenging tasks via code. It evaluates the true programming capabilities of large language models (LLMs) in a more realistic setting with diverse function calls from 139 popular libraries covering 723 API calls.\n\n## Task Description\n\n- **Task Type**: Code Generation (Python)\n- **Input**: Programming task description (docstring or natural language instruction)\n- **Output**: Complete Python function implementation\n- **Libraries**: 139 popular Python libraries (numpy, pandas, sklearn, etc.)\n\n## Key Features\n\n- 1,140 rich-context programming tasks in Python\n- Two evaluation modes: Complete (docstring) and Instruct (natural language)\n- Covers diverse function calls from 139 popular libraries\n- Uses unittest.TestCase for thorough correctness verification\n- Supports pass@k metric calculation\n\n## Evaluation Notes\n\n- **Sandbox Required**: Requires sandbox environment with 70+ Python libraries pre-installed\n- Two modes available via `split` parameter: `complete` (docstring completion) or `instruct` (NL instruction)\n- Default timeout is 240 seconds per problem\n- `calibrate` option prepends code_prompt to align function signatures\n- See [sandbox documentation](https://evalscope.readthedocs.io/en/latest/user_guides/sandbox.html) for setup\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `bigcodebench` |\n| **Dataset ID** | [evalscope/bigcodebench](https://modelscope.cn/datasets/evalscope/bigcodebench/summary) |\n| **Paper** | N/A |\n| **Tags** | `Coding` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `v0.1.4` |\n| **Aggregation** | `mean_and_pass_at_k` |\n\n\n## Data Statistics\n\n*Statistics not available.*\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"657d2473\",\n \"content\": \"Calculates the average of the sums of absolute differences between each pair of consecutive numbers for all permutations of a given list. Each permutation is shuffled before calculating the differences. Args: - numbers (list): A list of numbe ... [TRUNCATED 75 chars] ... loat: The average of the sums of absolute differences for each shuffled permutation of the list.\\nYou should write self-contained code starting with:\\n```\\nimport itertools\\nfrom random import shuffle\\ndef task_func(numbers=list(range(1, 3))):\\n```\"\n }\n ],\n \"target\": \" permutations = list(itertools.permutations(numbers))\\n sum_diffs = 0\\n\\n for perm in permutations:\\n perm = list(perm)\\n shuffle(perm)\\n diffs = [abs(perm[i] - perm[i+1]) for i in range(len(perm)-1)]\\n sum_diffs += sum(diffs)\\n\\n avg_sum_diffs = sum_diffs / len(permutations)\\n \\n return avg_sum_diffs\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"task_id\": \"BigCodeBench/0\",\n \"entry_point\": \"task_func\",\n \"complete_prompt\": \"import itertools\\nfrom random import shuffle\\n\\ndef task_func(numbers=list(range(1, 3))):\\n \\\"\\\"\\\"\\n Calculates the average of the sums of absolute differences between each pair of consecutive numbers \\n for all permutations of a given list. ... [TRUNCATED 187 chars] ... age of the sums of absolute differences for each shuffled permutation of the list.\\n\\n Requirements:\\n - itertools\\n - random.shuffle\\n\\n Example:\\n >>> result = task_func([1, 2, 3])\\n >>> isinstance(result, float)\\n True\\n \\\"\\\"\\\"\\n\",\n \"code_prompt\": \"import itertools\\nfrom random import shuffle\\ndef task_func(numbers=list(range(1, 3))):\\n\",\n \"test\": \"import unittest\\nfrom unittest.mock import patch\\nfrom random import seed, shuffle\\nimport itertools\\nclass TestCases(unittest.TestCase):\\n def test_default_numbers(self):\\n # Test with default number range (1 to 10) to check that the res ... [TRUNCATED 2578 chars] ... x: seed(1) or shuffle(x)):\\n result1 = task_func([1, 2, 3])\\n with patch('random.shuffle', side_effect=lambda x: seed(1) or shuffle(x)):\\n result2 = task_func([1, 2, 4])\\n self.assertNotEqual(result1, result2)\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{prompt}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `split` | `str` | `instruct` | Evaluation mode: \"complete\" (docstring completion) or \"instruct\" (NL instruction). Choices: ['complete', 'instruct'] |\n| `version` | `str` | `default` | Dataset version. Use \"default\" for the latest available version. |\n| `calibrate` | `bool` | `True` | Whether to prepend code_prompt to the solution for function signature alignment. |\n| `docker_build_context` | `str` | `` | Optional local Docker build context. When set, overrides the default sandbox image. |\n| `dockerfile` | `str` | `Dockerfile` | Dockerfile path inside docker_build_context. |\n| `force_rebuild` | `bool` | `False` | Force rebuilding the optional local Docker image. |\n\n## Sandbox Configuration\n\nThis benchmark requires a sandbox environment for code execution.\n\n```json\n{\n \"image\": \"bigcodebench/bigcodebench-evaluate:latest\",\n \"tools_config\": {\n \"shell_executor\": {},\n \"python_executor\": {}\n },\n \"memory_limit\": \"4g\"\n}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets bigcodebench \\\n --sandbox '{\"enabled\": true}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['bigcodebench'],\n sandbox={'enabled': True},\n dataset_args={\n 'bigcodebench': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# BigCodeBench\n\n\n## 概述\n\nBigCodeBench 是一个易于使用的基准测试，用于通过代码解决实际且具有挑战性的任务。它在更贴近现实的场景中评估大语言模型（LLMs）的真实编程能力，涵盖来自 139 个流行库的 723 个 API 调用。\n\n## 任务描述\n\n- **任务类型**：代码生成（Python）\n- **输入**：编程任务描述（文档字符串或自然语言指令）\n- **输出**：完整的 Python 函数实现\n- **库**：139 个流行的 Python 库（如 numpy、pandas、sklearn 等）\n\n## 主要特性\n\n- 包含 1,140 个上下文丰富的 Python 编程任务\n- 提供两种评估模式：Complete（基于文档字符串）和 Instruct（基于自然语言指令）\n- 覆盖来自 139 个流行库的多样化函数调用\n- 使用 `unittest.TestCase` 进行全面的正确性验证\n- 支持 pass@k 指标计算\n\n## 评估说明\n\n- **需要沙箱环境**：要求预装 70 多个 Python 库的沙箱环境\n- 通过 `split` 参数选择两种模式：`complete`（文档字符串补全）或 `instruct`（自然语言指令）\n- 每个问题默认超时时间为 240 秒\n- 启用 `calibrate` 选项会在生成代码前添加 `code_prompt`，以对齐函数签名\n- 沙箱环境设置详见 [沙箱文档](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/sandbox.html)\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `bigcodebench` |\n| **数据集ID** | [evalscope/bigcodebench](https://modelscope.cn/datasets/evalscope/bigcodebench/summary) |\n| **论文** | N/A |\n| **标签** | `Coding` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分版本** | `v0.1.4` |\n| **聚合方式** | `mean_and_pass_at_k` |\n\n\n## 数据统计\n\n*统计数据暂不可用。*\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"657d2473\",\n \"content\": \"Calculates the average of the sums of absolute differences between each pair of consecutive numbers for all permutations of a given list. Each permutation is shuffled before calculating the differences. Args: - numbers (list): A list of numbe ... [TRUNCATED 75 chars] ... loat: The average of the sums of absolute differences for each shuffled permutation of the list.\\nYou should write self-contained code starting with:\\n```\\nimport itertools\\nfrom random import shuffle\\ndef task_func(numbers=list(range(1, 3))):\\n```\"\n }\n ],\n \"target\": \" permutations = list(itertools.permutations(numbers))\\n sum_diffs = 0\\n\\n for perm in permutations:\\n perm = list(perm)\\n shuffle(perm)\\n diffs = [abs(perm[i] - perm[i+1]) for i in range(len(perm)-1)]\\n sum_diffs += sum(diffs)\\n\\n avg_sum_diffs = sum_diffs / len(permutations)\\n \\n return avg_sum_diffs\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"task_id\": \"BigCodeBench/0\",\n \"entry_point\": \"task_func\",\n \"complete_prompt\": \"import itertools\\nfrom random import shuffle\\n\\ndef task_func(numbers=list(range(1, 3))):\\n \\\"\\\"\\\"\\n Calculates the average of the sums of absolute differences between each pair of consecutive numbers \\n for all permutations of a given list. ... [TRUNCATED 187 chars] ... age of the sums of absolute differences for each shuffled permutation of the list.\\n\\n Requirements:\\n - itertools\\n - random.shuffle\\n\\n Example:\\n >>> result = task_func([1, 2, 3])\\n >>> isinstance(result, float)\\n True\\n \\\"\\\"\\\"\\n\",\n \"code_prompt\": \"import itertools\\nfrom random import shuffle\\ndef task_func(numbers=list(range(1, 3))):\\n\",\n \"test\": \"import unittest\\nfrom unittest.mock import patch\\nfrom random import seed, shuffle\\nimport itertools\\nclass TestCases(unittest.TestCase):\\n def test_default_numbers(self):\\n # Test with default number range (1 to 10) to check that the res ... [TRUNCATED 2578 chars] ... x: seed(1) or shuffle(x)):\\n result1 = task_func([1, 2, 3])\\n with patch('random.shuffle', side_effect=lambda x: seed(1) or shuffle(x)):\\n result2 = task_func([1, 2, 4])\\n self.assertNotEqual(result1, result2)\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{prompt}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `split` | `str` | `instruct` | 评估模式：\"complete\"（文档字符串补全）或 \"instruct\"（自然语言指令）。可选值：['complete', 'instruct'] |\n| `version` | `str` | `default` | 数据集版本。使用 \"default\" 表示最新可用版本。 |\n| `calibrate` | `bool` | `True` | 是否在解决方案前添加 `code_prompt` 以对齐函数签名。 |\n| `docker_build_context` | `str` | `` | 可选的本地 Docker 构建上下文。设置后将覆盖默认沙箱镜像。 |\n| `dockerfile` | `str` | `Dockerfile` | `docker_build_context` 内的 Dockerfile 路径。 |\n| `force_rebuild` | `bool` | `False` | 是否强制重建可选的本地 Docker 镜像。 |\n\n## 沙箱配置\n\n此基准测试需要沙箱环境来执行代码。\n\n```json\n{\n \"image\": \"bigcodebench/bigcodebench-evaluate:latest\",\n \"tools_config\": {\n \"shell_executor\": {},\n \"python_executor\": {}\n },\n \"memory_limit\": \"4g\"\n}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets bigcodebench \\\n --sandbox '{\"enabled\": true}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['bigcodebench'],\n sandbox={'enabled': True},\n dataset_args={\n 'bigcodebench': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "9647dba2918a5e2486443b2c41a188cf", "needs_translation": false }, - "updated_at": "2026-06-22T17:16:23.830036", - "translation_updated_at": "2026-06-22T17:16:34" -} \ No newline at end of file + "updated_at": "2026-07-07T17:13:28.622972", + "translation_updated_at": "2026-07-07T17:15:17" +} diff --git a/evalscope/evalscope/benchmarks/_meta/bigcodebench_hard.json b/evalscope/evalscope/benchmarks/_meta/bigcodebench_hard.json index 843a70b..16b9ca7 100644 --- a/evalscope/evalscope/benchmarks/_meta/bigcodebench_hard.json +++ b/evalscope/evalscope/benchmarks/_meta/bigcodebench_hard.json @@ -39,6 +39,21 @@ "type": "bool", "description": "Whether to prepend code_prompt to the solution for function signature alignment.", "value": true + }, + "docker_build_context": { + "type": "str", + "description": "Optional local Docker build context. When set, overrides the default sandbox image.", + "value": "" + }, + "dockerfile": { + "type": "str", + "description": "Dockerfile path inside docker_build_context.", + "value": "Dockerfile" + }, + "force_rebuild": { + "type": "bool", + "description": "Force rebuilding the optional local Docker image.", + "value": false } }, "sandbox_config": { @@ -96,11 +111,11 @@ "truncated": false }, "readme": { - "en": "# BigCodeBench-Hard\n\n\n## Overview\n\nBigCodeBench-Hard is a curated subset of BigCodeBench containing 148 tasks that are more aligned with real-world programming tasks. These tasks require more complex reasoning and multi-step problem solving.\n\n## Task Description\n\n- **Task Type**: Code Generation (Python)\n- **Input**: Programming task description (docstring or natural language instruction)\n- **Output**: Complete Python function implementation\n- **Difficulty**: Higher than BigCodeBench-Full, closer to real-world complexity\n\n## Key Features\n\n- 148 challenging tasks selected from BigCodeBench\n- Requires complex reasoning and multi-tool usage\n- Two evaluation modes: Complete (docstring) and Instruct (natural language)\n- Uses unittest.TestCase for thorough correctness verification\n- Supports pass@k metric calculation\n\n## Evaluation Notes\n\n- **Sandbox Required**: Requires sandbox environment with 70+ Python libraries pre-installed\n- Two modes available via `split` parameter: `complete` (docstring completion) or `instruct` (NL instruction)\n- Default timeout is 240 seconds per problem\n- `calibrate` option prepends code_prompt to align function signatures\n- See [sandbox documentation](https://evalscope.readthedocs.io/en/latest/user_guides/sandbox.html) for setup\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `bigcodebench_hard` |\n| **Dataset ID** | [evalscope/bigcodebench-hard](https://modelscope.cn/datasets/evalscope/bigcodebench-hard/summary) |\n| **Paper** | N/A |\n| **Tags** | `Coding` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `v0.1.4` |\n| **Aggregation** | `mean_and_pass_at_k` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 148 |\n| Prompt Length (Mean) | 777.94 chars |\n| Prompt Length (Min/Max) | 274 / 1816 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"207adc46\",\n \"content\": \"Download all files from a specific directory on an FTP server using wget in a subprocess. Args: ftp_server (str): The FTP server address. Default is 'ftp.dlptest.com'. ftp_user (str): The FTP server username. Default is 'dlpuser'. ftp_passwor ... [TRUNCATED 798 chars] ... FTP server.\\nYou should write self-contained code starting with:\\n```\\nimport subprocess\\nimport ftplib\\nimport os\\ndef task_func(ftp_server='ftp.dlptest.com', ftp_user='dlpuser', ftp_password='rNrKYTX9g7z3RgJRmxWuGHbeu', ftp_dir='/ftp/test'):\\n```\"\n }\n ],\n \"target\": \" # Attempt to connect to the FTP server\\n try:\\n ftp_obj = ftplib.FTP(ftp_server)\\n except Exception as e:\\n raise Exception(f'Failed to connect to FTP server {ftp_server}: {str(e)}')\\n\\n # Attempt to login to the FTP serv ... [TRUNCATED 625 chars] ... command = f'wget ftp://{ftp_user}:{ftp_password}@{ftp_server}{ftp_dir}/{filename} -P {download_dir}'\\n subprocess.call(command, shell=True)\\n downloaded_files.append(filename)\\n\\n ftp_obj.quit()\\n return downloaded_files\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"task_id\": \"BigCodeBench/13\",\n \"entry_point\": \"task_func\",\n \"complete_prompt\": \"import subprocess\\nimport ftplib\\nimport os\\n\\ndef task_func(ftp_server='ftp.dlptest.com', ftp_user='dlpuser', ftp_password='rNrKYTX9g7z3RgJRmxWuGHbeu', ftp_dir='/ftp/test'):\\n \\\"\\\"\\\"\\n Download all files from a specific directory on an FTP serv ... [TRUNCATED 909 chars] ... ctory. Outputs the message \\\"Failed to change to directory {ftp_dir} on server {ftp_server}: {str(e)}\\\"\\n \\n Requirements:\\n - subprocess\\n - ftplib\\n - os\\n\\n Example:\\n >>> task_func()\\n ['file1.txt', 'file2.jpg', ...]\\n \\\"\\\"\\\"\\n\",\n \"code_prompt\": \"import subprocess\\nimport ftplib\\nimport os\\ndef task_func(ftp_server='ftp.dlptest.com', ftp_user='dlpuser', ftp_password='rNrKYTX9g7z3RgJRmxWuGHbeu', ftp_dir='/ftp/test'):\\n\",\n \"test\": \"import unittest\\nfrom unittest.mock import patch\\nimport os\\nclass TestCases(unittest.TestCase):\\n def setUp(self):\\n \\\"\\\"\\\"Setup a clean test environment before each test.\\\"\\\"\\\"\\n if not os.path.exists(\\\"downloaded_files\\\"):\\n o ... [TRUNCATED 2565 chars] ... with self.assertRaises(Exception) as context:\\n task_func(ftp_dir=\\\"/invalid_directory\\\")\\n self.assertEqual(str(context.exception), f'Failed to change to directory /invalid_directory on server ftp.dlptest.com: {error_message}')\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{prompt}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `split` | `str` | `instruct` | Evaluation mode: \"complete\" (docstring completion) or \"instruct\" (NL instruction). Choices: ['complete', 'instruct'] |\n| `version` | `str` | `default` | Dataset version. Use \"default\" for the latest available version. |\n| `calibrate` | `bool` | `True` | Whether to prepend code_prompt to the solution for function signature alignment. |\n\n## Sandbox Configuration\n\nThis benchmark requires a sandbox environment for code execution.\n\n```json\n{\n \"image\": \"bigcodebench/bigcodebench-evaluate:latest\",\n \"tools_config\": {\n \"shell_executor\": {},\n \"python_executor\": {}\n },\n \"memory_limit\": \"4g\"\n}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets bigcodebench_hard \\\n --sandbox '{\"enabled\": true}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['bigcodebench_hard'],\n sandbox={'enabled': True},\n dataset_args={\n 'bigcodebench_hard': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# BigCodeBench-Hard\n\n\n## 概述\n\nBigCodeBench-Hard 是 BigCodeBench 的一个精选子集，包含 148 个更贴近真实编程场景的任务。这些任务需要更复杂的推理能力和多步骤的问题解决能力。\n\n## 任务描述\n\n- **任务类型**：代码生成（Python）\n- **输入**：编程任务描述（文档字符串或自然语言指令）\n- **输出**：完整的 Python 函数实现\n- **难度**：高于 BigCodeBench-Full，更接近真实世界的复杂度\n\n## 主要特性\n\n- 从 BigCodeBench 中精选出的 148 个具有挑战性的任务\n- 要求复杂的推理能力和多工具协同使用\n- 提供两种评估模式：Complete（基于文档字符串）和 Instruct（基于自然语言指令）\n- 使用 `unittest.TestCase` 进行全面的正确性验证\n- 支持 pass@k 指标计算\n\n## 评估说明\n\n- **需要沙箱环境**：要求预装 70 多个 Python 库的沙箱环境\n- 通过 `split` 参数选择两种模式：`complete`（文档字符串补全）或 `instruct`（自然语言指令）\n- 每个问题默认超时时间为 240 秒\n- 启用 `calibrate` 选项会将 `code_prompt` 添加到生成代码前，以对齐函数签名\n- 沙箱环境设置请参阅 [沙箱文档](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/sandbox.html)\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `bigcodebench_hard` |\n| **数据集ID** | [evalscope/bigcodebench-hard](https://modelscope.cn/datasets/evalscope/bigcodebench-hard/summary) |\n| **论文** | N/A |\n| **标签** | `Coding` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分版本** | `v0.1.4` |\n| **聚合方式** | `mean_and_pass_at_k` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 148 |\n| 提示词长度（平均） | 777.94 字符 |\n| 提示词长度（最小/最大） | 274 / 1816 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"207adc46\",\n \"content\": \"Download all files from a specific directory on an FTP server using wget in a subprocess. Args: ftp_server (str): The FTP server address. Default is 'ftp.dlptest.com'. ftp_user (str): The FTP server username. Default is 'dlpuser'. ftp_passwor ... [TRUNCATED 798 chars] ... FTP server.\\nYou should write self-contained code starting with:\\n```\\nimport subprocess\\nimport ftplib\\nimport os\\ndef task_func(ftp_server='ftp.dlptest.com', ftp_user='dlpuser', ftp_password='rNrKYTX9g7z3RgJRmxWuGHbeu', ftp_dir='/ftp/test'):\\n```\"\n }\n ],\n \"target\": \" # Attempt to connect to the FTP server\\n try:\\n ftp_obj = ftplib.FTP(ftp_server)\\n except Exception as e:\\n raise Exception(f'Failed to connect to FTP server {ftp_server}: {str(e)}')\\n\\n # Attempt to login to the FTP serv ... [TRUNCATED 625 chars] ... command = f'wget ftp://{ftp_user}:{ftp_password}@{ftp_server}{ftp_dir}/{filename} -P {download_dir}'\\n subprocess.call(command, shell=True)\\n downloaded_files.append(filename)\\n\\n ftp_obj.quit()\\n return downloaded_files\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"task_id\": \"BigCodeBench/13\",\n \"entry_point\": \"task_func\",\n \"complete_prompt\": \"import subprocess\\nimport ftplib\\nimport os\\n\\ndef task_func(ftp_server='ftp.dlptest.com', ftp_user='dlpuser', ftp_password='rNrKYTX9g7z3RgJRmxWuGHbeu', ftp_dir='/ftp/test'):\\n \\\"\\\"\\\"\\n Download all files from a specific directory on an FTP serv ... [TRUNCATED 909 chars] ... ctory. Outputs the message \\\"Failed to change to directory {ftp_dir} on server {ftp_server}: {str(e)}\\\"\\n \\n Requirements:\\n - subprocess\\n - ftplib\\n - os\\n\\n Example:\\n >>> task_func()\\n ['file1.txt', 'file2.jpg', ...]\\n \\\"\\\"\\\"\\n\",\n \"code_prompt\": \"import subprocess\\nimport ftplib\\nimport os\\ndef task_func(ftp_server='ftp.dlptest.com', ftp_user='dlpuser', ftp_password='rNrKYTX9g7z3RgJRmxWuGHbeu', ftp_dir='/ftp/test'):\\n\",\n \"test\": \"import unittest\\nfrom unittest.mock import patch\\nimport os\\nclass TestCases(unittest.TestCase):\\n def setUp(self):\\n \\\"\\\"\\\"Setup a clean test environment before each test.\\\"\\\"\\\"\\n if not os.path.exists(\\\"downloaded_files\\\"):\\n o ... [TRUNCATED 2565 chars] ... with self.assertRaises(Exception) as context:\\n task_func(ftp_dir=\\\"/invalid_directory\\\")\\n self.assertEqual(str(context.exception), f'Failed to change to directory /invalid_directory on server ftp.dlptest.com: {error_message}')\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板**：\n```text\n{prompt}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `split` | `str` | `instruct` | 评估模式：\"complete\"（文档字符串补全）或 \"instruct\"（自然语言指令）。可选值：['complete', 'instruct'] |\n| `version` | `str` | `default` | 数据集版本。使用 \"default\" 表示最新可用版本。 |\n| `calibrate` | `bool` | `True` | 是否在生成代码前添加 `code_prompt` 以对齐函数签名。 |\n\n## 沙箱配置\n\n此基准测试需要沙箱环境来执行代码。\n\n```json\n{\n \"image\": \"bigcodebench/bigcodebench-evaluate:latest\",\n \"tools_config\": {\n \"shell_executor\": {},\n \"python_executor\": {}\n },\n \"memory_limit\": \"4g\"\n}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets bigcodebench_hard \\\n --sandbox '{\"enabled\": true}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['bigcodebench_hard'],\n sandbox={'enabled': True},\n dataset_args={\n 'bigcodebench_hard': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "0414373dbf4c8fc19458129725e06ad8", + "en": "# BigCodeBench-Hard\n\n\n## Overview\n\nBigCodeBench-Hard is a curated subset of BigCodeBench containing 148 tasks that are more aligned with real-world programming tasks. These tasks require more complex reasoning and multi-step problem solving.\n\n## Task Description\n\n- **Task Type**: Code Generation (Python)\n- **Input**: Programming task description (docstring or natural language instruction)\n- **Output**: Complete Python function implementation\n- **Difficulty**: Higher than BigCodeBench-Full, closer to real-world complexity\n\n## Key Features\n\n- 148 challenging tasks selected from BigCodeBench\n- Requires complex reasoning and multi-tool usage\n- Two evaluation modes: Complete (docstring) and Instruct (natural language)\n- Uses unittest.TestCase for thorough correctness verification\n- Supports pass@k metric calculation\n\n## Evaluation Notes\n\n- **Sandbox Required**: Requires sandbox environment with 70+ Python libraries pre-installed\n- Two modes available via `split` parameter: `complete` (docstring completion) or `instruct` (NL instruction)\n- Default timeout is 240 seconds per problem\n- `calibrate` option prepends code_prompt to align function signatures\n- See [sandbox documentation](https://evalscope.readthedocs.io/en/latest/user_guides/sandbox.html) for setup\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `bigcodebench_hard` |\n| **Dataset ID** | [evalscope/bigcodebench-hard](https://modelscope.cn/datasets/evalscope/bigcodebench-hard/summary) |\n| **Paper** | N/A |\n| **Tags** | `Coding` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `v0.1.4` |\n| **Aggregation** | `mean_and_pass_at_k` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 148 |\n| Prompt Length (Mean) | 777.94 chars |\n| Prompt Length (Min/Max) | 274 / 1816 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"207adc46\",\n \"content\": \"Download all files from a specific directory on an FTP server using wget in a subprocess. Args: ftp_server (str): The FTP server address. Default is 'ftp.dlptest.com'. ftp_user (str): The FTP server username. Default is 'dlpuser'. ftp_passwor ... [TRUNCATED 798 chars] ... FTP server.\\nYou should write self-contained code starting with:\\n```\\nimport subprocess\\nimport ftplib\\nimport os\\ndef task_func(ftp_server='ftp.dlptest.com', ftp_user='dlpuser', ftp_password='rNrKYTX9g7z3RgJRmxWuGHbeu', ftp_dir='/ftp/test'):\\n```\"\n }\n ],\n \"target\": \" # Attempt to connect to the FTP server\\n try:\\n ftp_obj = ftplib.FTP(ftp_server)\\n except Exception as e:\\n raise Exception(f'Failed to connect to FTP server {ftp_server}: {str(e)}')\\n\\n # Attempt to login to the FTP serv ... [TRUNCATED 625 chars] ... command = f'wget ftp://{ftp_user}:{ftp_password}@{ftp_server}{ftp_dir}/{filename} -P {download_dir}'\\n subprocess.call(command, shell=True)\\n downloaded_files.append(filename)\\n\\n ftp_obj.quit()\\n return downloaded_files\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"task_id\": \"BigCodeBench/13\",\n \"entry_point\": \"task_func\",\n \"complete_prompt\": \"import subprocess\\nimport ftplib\\nimport os\\n\\ndef task_func(ftp_server='ftp.dlptest.com', ftp_user='dlpuser', ftp_password='rNrKYTX9g7z3RgJRmxWuGHbeu', ftp_dir='/ftp/test'):\\n \\\"\\\"\\\"\\n Download all files from a specific directory on an FTP serv ... [TRUNCATED 909 chars] ... ctory. Outputs the message \\\"Failed to change to directory {ftp_dir} on server {ftp_server}: {str(e)}\\\"\\n \\n Requirements:\\n - subprocess\\n - ftplib\\n - os\\n\\n Example:\\n >>> task_func()\\n ['file1.txt', 'file2.jpg', ...]\\n \\\"\\\"\\\"\\n\",\n \"code_prompt\": \"import subprocess\\nimport ftplib\\nimport os\\ndef task_func(ftp_server='ftp.dlptest.com', ftp_user='dlpuser', ftp_password='rNrKYTX9g7z3RgJRmxWuGHbeu', ftp_dir='/ftp/test'):\\n\",\n \"test\": \"import unittest\\nfrom unittest.mock import patch\\nimport os\\nclass TestCases(unittest.TestCase):\\n def setUp(self):\\n \\\"\\\"\\\"Setup a clean test environment before each test.\\\"\\\"\\\"\\n if not os.path.exists(\\\"downloaded_files\\\"):\\n o ... [TRUNCATED 2565 chars] ... with self.assertRaises(Exception) as context:\\n task_func(ftp_dir=\\\"/invalid_directory\\\")\\n self.assertEqual(str(context.exception), f'Failed to change to directory /invalid_directory on server ftp.dlptest.com: {error_message}')\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{prompt}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `split` | `str` | `instruct` | Evaluation mode: \"complete\" (docstring completion) or \"instruct\" (NL instruction). Choices: ['complete', 'instruct'] |\n| `version` | `str` | `default` | Dataset version. Use \"default\" for the latest available version. |\n| `calibrate` | `bool` | `True` | Whether to prepend code_prompt to the solution for function signature alignment. |\n| `docker_build_context` | `str` | `` | Optional local Docker build context. When set, overrides the default sandbox image. |\n| `dockerfile` | `str` | `Dockerfile` | Dockerfile path inside docker_build_context. |\n| `force_rebuild` | `bool` | `False` | Force rebuilding the optional local Docker image. |\n\n## Sandbox Configuration\n\nThis benchmark requires a sandbox environment for code execution.\n\n```json\n{\n \"image\": \"bigcodebench/bigcodebench-evaluate:latest\",\n \"tools_config\": {\n \"shell_executor\": {},\n \"python_executor\": {}\n },\n \"memory_limit\": \"4g\"\n}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets bigcodebench_hard \\\n --sandbox '{\"enabled\": true}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['bigcodebench_hard'],\n sandbox={'enabled': True},\n dataset_args={\n 'bigcodebench_hard': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# BigCodeBench-Hard\n\n\n## 概述\n\nBigCodeBench-Hard 是 BigCodeBench 的一个精选子集，包含 148 个更贴近真实编程场景的任务。这些任务需要更复杂的推理能力和多步骤的问题解决能力。\n\n## 任务描述\n\n- **任务类型**：代码生成（Python）\n- **输入**：编程任务描述（文档字符串或自然语言指令）\n- **输出**：完整的 Python 函数实现\n- **难度**：高于 BigCodeBench-Full，更接近真实世界的复杂度\n\n## 主要特性\n\n- 从 BigCodeBench 中精选出的 148 个具有挑战性的任务\n- 要求复杂的推理能力和多工具协同使用\n- 提供两种评估模式：Complete（文档字符串补全）和 Instruct（自然语言指令）\n- 使用 `unittest.TestCase` 进行全面的正确性验证\n- 支持 pass@k 指标计算\n\n## 评估说明\n\n- **需要沙箱环境**：要求预装 70 多个 Python 库的沙箱环境\n- 通过 `split` 参数可选择两种模式：`complete`（文档字符串补全）或 `instruct`（自然语言指令）\n- 每个问题默认超时时间为 240 秒\n- 启用 `calibrate` 选项会在生成代码前添加 `code_prompt`，以对齐函数签名\n- 沙箱环境设置详见 [沙箱文档](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/sandbox.html)\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `bigcodebench_hard` |\n| **数据集ID** | [evalscope/bigcodebench-hard](https://modelscope.cn/datasets/evalscope/bigcodebench-hard/summary) |\n| **论文** | N/A |\n| **标签** | `Coding` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分版本** | `v0.1.4` |\n| **聚合方式** | `mean_and_pass_at_k` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 148 |\n| 提示词长度（平均） | 777.94 字符 |\n| 提示词长度（最小/最大） | 274 / 1816 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"207adc46\",\n \"content\": \"Download all files from a specific directory on an FTP server using wget in a subprocess. Args: ftp_server (str): The FTP server address. Default is 'ftp.dlptest.com'. ftp_user (str): The FTP server username. Default is 'dlpuser'. ftp_passwor ... [TRUNCATED 798 chars] ... FTP server.\\nYou should write self-contained code starting with:\\n```\\nimport subprocess\\nimport ftplib\\nimport os\\ndef task_func(ftp_server='ftp.dlptest.com', ftp_user='dlpuser', ftp_password='rNrKYTX9g7z3RgJRmxWuGHbeu', ftp_dir='/ftp/test'):\\n```\"\n }\n ],\n \"target\": \" # Attempt to connect to the FTP server\\n try:\\n ftp_obj = ftplib.FTP(ftp_server)\\n except Exception as e:\\n raise Exception(f'Failed to connect to FTP server {ftp_server}: {str(e)}')\\n\\n # Attempt to login to the FTP serv ... [TRUNCATED 625 chars] ... command = f'wget ftp://{ftp_user}:{ftp_password}@{ftp_server}{ftp_dir}/{filename} -P {download_dir}'\\n subprocess.call(command, shell=True)\\n downloaded_files.append(filename)\\n\\n ftp_obj.quit()\\n return downloaded_files\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"task_id\": \"BigCodeBench/13\",\n \"entry_point\": \"task_func\",\n \"complete_prompt\": \"import subprocess\\nimport ftplib\\nimport os\\n\\ndef task_func(ftp_server='ftp.dlptest.com', ftp_user='dlpuser', ftp_password='rNrKYTX9g7z3RgJRmxWuGHbeu', ftp_dir='/ftp/test'):\\n \\\"\\\"\\\"\\n Download all files from a specific directory on an FTP serv ... [TRUNCATED 909 chars] ... ctory. Outputs the message \\\"Failed to change to directory {ftp_dir} on server {ftp_server}: {str(e)}\\\"\\n \\n Requirements:\\n - subprocess\\n - ftplib\\n - os\\n\\n Example:\\n >>> task_func()\\n ['file1.txt', 'file2.jpg', ...]\\n \\\"\\\"\\\"\\n\",\n \"code_prompt\": \"import subprocess\\nimport ftplib\\nimport os\\ndef task_func(ftp_server='ftp.dlptest.com', ftp_user='dlpuser', ftp_password='rNrKYTX9g7z3RgJRmxWuGHbeu', ftp_dir='/ftp/test'):\\n\",\n \"test\": \"import unittest\\nfrom unittest.mock import patch\\nimport os\\nclass TestCases(unittest.TestCase):\\n def setUp(self):\\n \\\"\\\"\\\"Setup a clean test environment before each test.\\\"\\\"\\\"\\n if not os.path.exists(\\\"downloaded_files\\\"):\\n o ... [TRUNCATED 2565 chars] ... with self.assertRaises(Exception) as context:\\n task_func(ftp_dir=\\\"/invalid_directory\\\")\\n self.assertEqual(str(context.exception), f'Failed to change to directory /invalid_directory on server ftp.dlptest.com: {error_message}')\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{prompt}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `split` | `str` | `instruct` | 评估模式：\"complete\"（文档字符串补全）或 \"instruct\"（自然语言指令）。可选值：['complete', 'instruct'] |\n| `version` | `str` | `default` | 数据集版本。使用 \"default\" 表示最新可用版本。 |\n| `calibrate` | `bool` | `True` | 是否在解决方案前添加 `code_prompt` 以对齐函数签名。 |\n| `docker_build_context` | `str` | `` | 可选的本地 Docker 构建上下文。设置后将覆盖默认沙箱镜像。 |\n| `dockerfile` | `str` | `Dockerfile` | `docker_build_context` 内的 Dockerfile 路径。 |\n| `force_rebuild` | `bool` | `False` | 是否强制重建可选的本地 Docker 镜像。 |\n\n## 沙箱配置\n\n此基准测试需要沙箱环境来执行代码。\n\n```json\n{\n \"image\": \"bigcodebench/bigcodebench-evaluate:latest\",\n \"tools_config\": {\n \"shell_executor\": {},\n \"python_executor\": {}\n },\n \"memory_limit\": \"4g\"\n}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets bigcodebench_hard \\\n --sandbox '{\"enabled\": true}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['bigcodebench_hard'],\n sandbox={'enabled': True},\n dataset_args={\n 'bigcodebench_hard': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "08cc869d7c94d9122e6f33d47a3f7a3f", "needs_translation": false }, - "updated_at": "2026-06-22T17:16:20.214818", - "translation_updated_at": "2026-06-22T17:16:34" -} \ No newline at end of file + "updated_at": "2026-07-07T17:13:28.625055", + "translation_updated_at": "2026-07-07T17:15:17" +} diff --git a/evalscope/evalscope/benchmarks/_meta/biomix_qa.json b/evalscope/evalscope/benchmarks/_meta/biomix_qa.json index ee4f887..258387f 100644 --- a/evalscope/evalscope/benchmarks/_meta/biomix_qa.json +++ b/evalscope/evalscope/benchmarks/_meta/biomix_qa.json @@ -79,4 +79,4 @@ }, "updated_at": "2026-01-28T17:31:32.192148", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/blink.json b/evalscope/evalscope/benchmarks/_meta/blink.json index 4e8cec6..e63b323 100644 --- a/evalscope/evalscope/benchmarks/_meta/blink.json +++ b/evalscope/evalscope/benchmarks/_meta/blink.json @@ -671,4 +671,4 @@ }, "updated_at": "2026-01-28T17:31:32.192814", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/broad_twitter_corpus.json b/evalscope/evalscope/benchmarks/_meta/broad_twitter_corpus.json index d804842..9218e0e 100644 --- a/evalscope/evalscope/benchmarks/_meta/broad_twitter_corpus.json +++ b/evalscope/evalscope/benchmarks/_meta/broad_twitter_corpus.json @@ -111,4 +111,4 @@ }, "updated_at": "2026-01-28T17:31:32.340322", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/browsecomp.json b/evalscope/evalscope/benchmarks/_meta/browsecomp.json index b0dffd0..907eb32 100644 --- a/evalscope/evalscope/benchmarks/_meta/browsecomp.json +++ b/evalscope/evalscope/benchmarks/_meta/browsecomp.json @@ -58,4 +58,4 @@ }, "updated_at": "2026-06-17T22:07:18.747702", "translation_updated_at": "2026-06-17T22:07:21" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/cc_bench.json b/evalscope/evalscope/benchmarks/_meta/cc_bench.json index 86b16c8..652db10 100644 --- a/evalscope/evalscope/benchmarks/_meta/cc_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/cc_bench.json @@ -153,4 +153,4 @@ }, "updated_at": "2026-01-28T17:31:32.299360", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/ceval.json b/evalscope/evalscope/benchmarks/_meta/ceval.json index e708736..0427ea6 100644 --- a/evalscope/evalscope/benchmarks/_meta/ceval.json +++ b/evalscope/evalscope/benchmarks/_meta/ceval.json @@ -592,4 +592,4 @@ }, "updated_at": "2026-01-28T17:31:32.194461", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/chartqa.json b/evalscope/evalscope/benchmarks/_meta/chartqa.json index 7317acb..07d1292 100644 --- a/evalscope/evalscope/benchmarks/_meta/chartqa.json +++ b/evalscope/evalscope/benchmarks/_meta/chartqa.json @@ -185,4 +185,4 @@ }, "updated_at": "2026-01-28T17:31:32.195098", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/charxiv.json b/evalscope/evalscope/benchmarks/_meta/charxiv.json index e047743..fb722b4 100644 --- a/evalscope/evalscope/benchmarks/_meta/charxiv.json +++ b/evalscope/evalscope/benchmarks/_meta/charxiv.json @@ -191,4 +191,4 @@ }, "updated_at": "2026-07-03T11:09:49.196684", "translation_updated_at": "2026-07-03T16:19:29" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/chinese_simpleqa.json b/evalscope/evalscope/benchmarks/_meta/chinese_simpleqa.json index 1b40d0d..a2cdf2e 100644 --- a/evalscope/evalscope/benchmarks/_meta/chinese_simpleqa.json +++ b/evalscope/evalscope/benchmarks/_meta/chinese_simpleqa.json @@ -129,4 +129,4 @@ }, "updated_at": "2026-01-28T17:31:32.200945", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/cl_bench.json b/evalscope/evalscope/benchmarks/_meta/cl_bench.json index 872b42e..9a8ae52 100644 --- a/evalscope/evalscope/benchmarks/_meta/cl_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/cl_bench.json @@ -93,4 +93,4 @@ }, "updated_at": "2026-02-07T10:36:33.238019", "translation_updated_at": "2026-02-07T10:36:51Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/claw_eval.json b/evalscope/evalscope/benchmarks/_meta/claw_eval.json new file mode 100644 index 0000000..f12044e --- /dev/null +++ b/evalscope/evalscope/benchmarks/_meta/claw_eval.json @@ -0,0 +1,112 @@ +{ + "meta": { + "pretty_name": "Claw-Eval", + "dataset_id": "claw-eval/Claw-Eval", + "paper_url": null, + "tags": [ + "Agent", + "MultiModal", + "MultiTurn" + ], + "metrics": [ + "avg_score", + "pass_at_k", + "pass_hat_k", + "error_rate" + ], + "few_shot_num": 0, + "eval_split": "test", + "train_split": "", + "subset_list": [ + "general", + "multimodal", + "multi_turn" + ], + "description": "\n## Overview\n\nClaw-Eval evaluates assistant agents on realistic personal-assistant workflows that require tool use, file and fixture\naccess, multimodal inputs, and simulated user interactions. EvalScope runs the pinned official Claw-Eval Python runner,\nDocker sandbox, and graders while exposing each Claw-Eval task as a normal EvalScope sample for caching, repeats,\nparallel execution, reporting, and dashboard trace review.\n\n## Task Description\n\n- **Task Type**: Agentic personal-assistant tasks with tool use, sandbox files, multimodal fixtures, and optional\n simulated user turns.\n- **Dataset**: `claw-eval/Claw-Eval` on ModelScope.\n- **Subsets**: `general`, `multimodal`, and `multi_turn`; the current ModelScope manifest contains 300 tasks\n (161 general, 101 multimodal, and 38 multi_turn). Use `subset_list` to select subsets.\n- **Output**: Official Claw-Eval scores and JSONL traces, EvalScope sample-level reviews, grouped summary metrics, and\n dashboard-rendered agent traces.\n\n## Evaluation Notes\n\n- Requires Python 3.11+ and the official package installed from the pinned source commit:\n `pip install \"claw-eval[sandbox,mock,web] @\n git+https://github.com/claw-eval/claw-eval.git@d3f02d4938ab0832377d90535013def2b1a2fdc0\"`.\n- The installed package provides the Claw-Eval runner APIs. EvalScope also caches the same pinned source archive because\n `tasks/` and `Dockerfile.agent` are runtime assets, then loads the task manifest and fixtures from ModelScope.\n- Full fixtures are downloaded from ModelScope (`data/fixtures.tar.gz`) and linked into the official task tree before\n execution. The archive is large; use `limit` or `extra_params.task_ids` for smoke runs.\n- Each selected Claw-Eval task is one EvalScope sample. Official scoring runs once per sample; use EvalScope `repeats`\n for repeated trials per task and `eval_batch_size` for task-level worker concurrency.\n- Claw-Eval runs with the official Docker sandbox image. If `claw-eval-agent:latest` is missing locally, EvalScope\n builds it automatically from the cached official `Dockerfile.agent`. The first run can be slow.\n- EvalScope `use_cache` resumes completed task-level samples. Claw-Eval trace JSONL files are stored under\n `outputs/.../claw_eval//traces` and converted to EvalScope agent traces for dashboard visualization.\n", + "prompt_template": "{question}", + "system_prompt": "", + "few_shot_prompt_template": "", + "aggregation": "mean", + "extra_params": { + "task_ids": { + "type": "list", + "description": "Optional exact Claw-Eval task ids to run after split filtering.", + "value": [] + } + }, + "sandbox_config": {}, + "category": "agent" + }, + "statistics": { + "total_samples": 300, + "subset_stats": [ + { + "name": "general", + "sample_count": 161, + "prompt_length_mean": 46.47, + "prompt_length_min": 34, + "prompt_length_max": 58, + "prompt_length_std": 4.87, + "target_length_mean": null + }, + { + "name": "multimodal", + "sample_count": 101, + "prompt_length_mean": 49.75, + "prompt_length_min": 30, + "prompt_length_max": 60, + "prompt_length_std": 5.97, + "target_length_mean": null + }, + { + "name": "multi_turn", + "sample_count": 38, + "prompt_length_mean": 44.79, + "prompt_length_min": 35, + "prompt_length_max": 51, + "prompt_length_std": 3.68, + "target_length_mean": null + } + ], + "prompt_length": { + "mean": 47.36, + "min": 30, + "max": 60, + "std": 5.43 + }, + "target_length_mean": null, + "computed_at": "2026-07-14T16:59:09.900658" + }, + "sample_example": { + "data": { + "input": [ + { + "id": "a34b7f6b", + "content": "Run Claw-Eval task T001zh_email_triage." + } + ], + "target": "", + "id": 0, + "group_id": 0, + "subset_key": "general", + "metadata": { + "task_id": "T001zh_email_triage", + "split": "general", + "task_name": "", + "difficulty": "", + "dataset_id": "claw-eval/Claw-Eval", + "dataset_hub": "modelscope" + } + }, + "subset": "general", + "truncated": false + }, + "readme": { + "en": "# Claw-Eval\n\n\n## Overview\n\nClaw-Eval evaluates assistant agents on realistic personal-assistant workflows that require tool use, file and fixture\naccess, multimodal inputs, and simulated user interactions. EvalScope runs the pinned official Claw-Eval Python runner,\nDocker sandbox, and graders while exposing each Claw-Eval task as a normal EvalScope sample for caching, repeats,\nparallel execution, reporting, and dashboard trace review.\n\n## Task Description\n\n- **Task Type**: Agentic personal-assistant tasks with tool use, sandbox files, multimodal fixtures, and optional\n simulated user turns.\n- **Dataset**: `claw-eval/Claw-Eval` on ModelScope.\n- **Subsets**: `general`, `multimodal`, and `multi_turn`; the current ModelScope manifest contains 300 tasks\n (161 general, 101 multimodal, and 38 multi_turn). Use `subset_list` to select subsets.\n- **Output**: Official Claw-Eval scores and JSONL traces, EvalScope sample-level reviews, grouped summary metrics, and\n dashboard-rendered agent traces.\n\n## Evaluation Notes\n\n- Requires Python 3.11+ and the official package installed from the pinned source commit:\n `pip install \"claw-eval[sandbox,mock,web] @\n git+https://github.com/claw-eval/claw-eval.git@d3f02d4938ab0832377d90535013def2b1a2fdc0\"`.\n- The installed package provides the Claw-Eval runner APIs. EvalScope also caches the same pinned source archive because\n `tasks/` and `Dockerfile.agent` are runtime assets, then loads the task manifest and fixtures from ModelScope.\n- Full fixtures are downloaded from ModelScope (`data/fixtures.tar.gz`) and linked into the official task tree before\n execution. The archive is large; use `limit` or `extra_params.task_ids` for smoke runs.\n- Each selected Claw-Eval task is one EvalScope sample. Official scoring runs once per sample; use EvalScope `repeats`\n for repeated trials per task and `eval_batch_size` for task-level worker concurrency.\n- Claw-Eval runs with the official Docker sandbox image. If `claw-eval-agent:latest` is missing locally, EvalScope\n builds it automatically from the cached official `Dockerfile.agent`. The first run can be slow.\n- EvalScope `use_cache` resumes completed task-level samples. Claw-Eval trace JSONL files are stored under\n `outputs/.../claw_eval//traces` and converted to EvalScope agent traces for dashboard visualization.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `claw_eval` |\n| **Dataset ID** | [claw-eval/Claw-Eval](https://modelscope.cn/datasets/claw-eval/Claw-Eval/summary) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `MultiModal`, `MultiTurn` |\n| **Metrics** | `avg_score`, `pass_at_k`, `pass_hat_k`, `error_rate` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 300 |\n| Prompt Length (Mean) | 47.36 chars |\n| Prompt Length (Min/Max) | 30 / 60 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `general` | 161 | 46.47 | 34 | 58 |\n| `multimodal` | 101 | 49.75 | 30 | 60 |\n| `multi_turn` | 38 | 44.79 | 35 | 51 |\n\n## Sample Example\n\n**Subset**: `general`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"a34b7f6b\",\n \"content\": \"Run Claw-Eval task T001zh_email_triage.\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"general\",\n \"metadata\": {\n \"task_id\": \"T001zh_email_triage\",\n \"split\": \"general\",\n \"task_name\": \"\",\n \"difficulty\": \"\",\n \"dataset_id\": \"claw-eval/Claw-Eval\",\n \"dataset_hub\": \"modelscope\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `task_ids` | `list` | `[]` | Optional exact Claw-Eval task ids to run after split filtering. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets claw_eval \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['claw_eval'],\n dataset_args={\n 'claw_eval': {\n # subset_list: ['general', 'multimodal', 'multi_turn'] # optional, evaluate specific subsets\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# Claw-Eval\n\n\n## 概述\n\nClaw-Eval 用于评估助手代理在真实个人助理工作流中的表现，这些工作流需要使用工具、访问文件和固定资源（fixtures）、处理多模态输入，并支持模拟用户交互。EvalScope 运行官方指定版本的 Claw-Eval Python 执行器、Docker 沙箱和评分器，同时将每个 Claw-Eval 任务作为标准的 EvalScope 样本进行处理，以支持缓存、重复执行、并行运行、报告生成以及仪表盘轨迹审查。\n\n## 任务描述\n\n- **任务类型**：涉及工具调用、沙箱文件、多模态固定资源及可选模拟用户轮次的代理型个人助理任务。\n- **数据集**：ModelScope 上的 `claw-eval/Claw-Eval`。\n- **子集**：`general`、`multimodal` 和 `multi_turn`；当前 ModelScope 清单包含 300 个任务（161 个 general、101 个 multimodal、38 个 multi_turn）。可通过 `subset_list` 选择子集。\n- **输出**：官方 Claw-Eval 评分与 JSONL 轨迹、EvalScope 样本级评审结果、分组汇总指标，以及仪表盘渲染的代理轨迹。\n\n## 评估说明\n\n- 需要 Python 3.11+，并从指定源代码提交安装官方包：\n `pip install \"claw-eval[sandbox,mock,web] @ git+https://github.com/claw-eval/claw-eval.git@d3f02d4938ab0832377d90535013def2b1a2fdc0\"`。\n- 安装的包提供 Claw-Eval 执行器 API。EvalScope 同时缓存相同版本的源代码归档，因为 `tasks/` 和 `Dockerfile.agent` 是运行时资产，随后从 ModelScope 加载任务清单和固定资源。\n- 完整的固定资源从 ModelScope 下载（`data/fixtures.tar.gz`），并在执行前链接到官方任务目录中。该归档较大；建议在试运行时使用 `limit` 或 `extra_params.task_ids` 参数。\n- 每个选定的 Claw-Eval 任务对应一个 EvalScope 样本。官方评分对每个样本仅运行一次；如需对同一任务进行多次试验，请使用 EvalScope 的 `repeats` 参数；如需任务级并发执行，请使用 `eval_batch_size`。\n- Claw-Eval 使用官方 Docker 沙箱镜像运行。若本地缺少 `claw-eval-agent:latest` 镜像，EvalScope 会自动基于缓存的官方 `Dockerfile.agent` 构建。首次运行可能较慢。\n- EvalScope 的 `use_cache` 功能可恢复已完成的任务级样本。Claw-Eval 轨迹 JSONL 文件存储在 `outputs/.../claw_eval//traces` 目录下，并转换为 EvalScope 代理轨迹以供仪表盘可视化。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `claw_eval` |\n| **数据集ID** | [claw-eval/Claw-Eval](https://modelscope.cn/datasets/claw-eval/Claw-Eval/summary) |\n| **论文** | 无 |\n| **标签** | `Agent`, `MultiModal`, `MultiTurn` |\n| **指标** | `avg_score`, `pass_at_k`, `pass_hat_k`, `error_rate` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 300 |\n| 提示词长度（平均） | 47.36 字符 |\n| 提示词长度（最小/最大） | 30 / 60 字符 |\n\n**各子集统计数据：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `general` | 161 | 46.47 | 34 | 58 |\n| `multimodal` | 101 | 49.75 | 30 | 60 |\n| `multi_turn` | 38 | 44.79 | 35 | 51 |\n\n## 样例示例\n\n**子集**: `general`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"a34b7f6b\",\n \"content\": \"Run Claw-Eval task T001zh_email_triage.\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"general\",\n \"metadata\": {\n \"task_id\": \"T001zh_email_triage\",\n \"split\": \"general\",\n \"task_name\": \"\",\n \"difficulty\": \"\",\n \"dataset_id\": \"claw-eval/Claw-Eval\",\n \"dataset_hub\": \"modelscope\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `task_ids` | `list` | `[]` | 可选，在子集过滤后精确指定要运行的 Claw-Eval 任务 ID 列表。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets claw_eval \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['claw_eval'],\n dataset_args={\n 'claw_eval': {\n # subset_list: ['general', 'multimodal', 'multi_turn'] # 可选，评估特定子集\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "96fad09d5529737f9cd0025e2494c9cd", + "needs_translation": false + }, + "updated_at": "2026-07-14T17:44:31.169240", + "translation_updated_at": "2026-07-14T17:44:38" +} diff --git a/evalscope/evalscope/benchmarks/_meta/cmath.json b/evalscope/evalscope/benchmarks/_meta/cmath.json index 7ed546c..6f0a10a 100644 --- a/evalscope/evalscope/benchmarks/_meta/cmath.json +++ b/evalscope/evalscope/benchmarks/_meta/cmath.json @@ -130,4 +130,4 @@ }, "updated_at": "2026-07-03T16:31:10.424720", "translation_updated_at": "2026-07-03T16:31:16" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/cmmlu.json b/evalscope/evalscope/benchmarks/_meta/cmmlu.json index ec16d96..17583fc 100644 --- a/evalscope/evalscope/benchmarks/_meta/cmmlu.json +++ b/evalscope/evalscope/benchmarks/_meta/cmmlu.json @@ -741,4 +741,4 @@ }, "updated_at": "2026-01-28T17:31:32.203370", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/cmmmu.json b/evalscope/evalscope/benchmarks/_meta/cmmmu.json index 2d67c81..e04d0fc 100644 --- a/evalscope/evalscope/benchmarks/_meta/cmmmu.json +++ b/evalscope/evalscope/benchmarks/_meta/cmmmu.json @@ -1399,4 +1399,4 @@ }, "updated_at": "2026-01-28T17:31:32.203423", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/cmmu.json b/evalscope/evalscope/benchmarks/_meta/cmmu.json index 28a5a9c..42e72e1 100644 --- a/evalscope/evalscope/benchmarks/_meta/cmmu.json +++ b/evalscope/evalscope/benchmarks/_meta/cmmu.json @@ -431,4 +431,4 @@ }, "updated_at": "2026-01-28T17:31:32.203937", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/coin_flip.json b/evalscope/evalscope/benchmarks/_meta/coin_flip.json index 8aa9e79..6c836a2 100644 --- a/evalscope/evalscope/benchmarks/_meta/coin_flip.json +++ b/evalscope/evalscope/benchmarks/_meta/coin_flip.json @@ -81,4 +81,4 @@ }, "updated_at": "2026-01-28T17:31:32.204932", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/common_voice_15.json b/evalscope/evalscope/benchmarks/_meta/common_voice_15.json index c1bc5ce..d48bc9a 100644 --- a/evalscope/evalscope/benchmarks/_meta/common_voice_15.json +++ b/evalscope/evalscope/benchmarks/_meta/common_voice_15.json @@ -177,4 +177,4 @@ }, "updated_at": "2026-06-23T15:37:29.293476+08:00", "translation_updated_at": "2026-06-23T15:50:44+08:00" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/commonsense_qa.json b/evalscope/evalscope/benchmarks/_meta/commonsense_qa.json index 452300c..affc938 100644 --- a/evalscope/evalscope/benchmarks/_meta/commonsense_qa.json +++ b/evalscope/evalscope/benchmarks/_meta/commonsense_qa.json @@ -79,4 +79,4 @@ }, "updated_at": "2026-01-28T17:31:32.207495", "translation_updated_at": "2026-01-28T15:56:15Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/competition_math.json b/evalscope/evalscope/benchmarks/_meta/competition_math.json index 30ef4ca..52c7fde 100644 --- a/evalscope/evalscope/benchmarks/_meta/competition_math.json +++ b/evalscope/evalscope/benchmarks/_meta/competition_math.json @@ -119,4 +119,4 @@ }, "updated_at": "2026-01-28T17:31:32.213604", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/conll2003.json b/evalscope/evalscope/benchmarks/_meta/conll2003.json index e54fc45..e642d48 100644 --- a/evalscope/evalscope/benchmarks/_meta/conll2003.json +++ b/evalscope/evalscope/benchmarks/_meta/conll2003.json @@ -103,4 +103,4 @@ }, "updated_at": "2026-01-28T17:31:32.340919", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/conllpp.json b/evalscope/evalscope/benchmarks/_meta/conllpp.json index c642d00..f43099e 100644 --- a/evalscope/evalscope/benchmarks/_meta/conllpp.json +++ b/evalscope/evalscope/benchmarks/_meta/conllpp.json @@ -103,4 +103,4 @@ }, "updated_at": "2026-01-28T17:31:32.347147", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/copious.json b/evalscope/evalscope/benchmarks/_meta/copious.json index 78284c3..63c68fd 100644 --- a/evalscope/evalscope/benchmarks/_meta/copious.json +++ b/evalscope/evalscope/benchmarks/_meta/copious.json @@ -909,4 +909,4 @@ }, "updated_at": "2026-01-28T17:31:32.350016", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/cross_ner.json b/evalscope/evalscope/benchmarks/_meta/cross_ner.json index 53a8bcc..d621b9a 100644 --- a/evalscope/evalscope/benchmarks/_meta/cross_ner.json +++ b/evalscope/evalscope/benchmarks/_meta/cross_ner.json @@ -159,4 +159,4 @@ }, "updated_at": "2026-01-28T17:31:32.352927", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/data_collection.json b/evalscope/evalscope/benchmarks/_meta/data_collection.json index aa5fe4e..b02f06a 100644 --- a/evalscope/evalscope/benchmarks/_meta/data_collection.json +++ b/evalscope/evalscope/benchmarks/_meta/data_collection.json @@ -45,4 +45,4 @@ }, "updated_at": "2026-01-28T17:31:32.213349", "translation_updated_at": "2026-03-05T17:50:46Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/deep_swe.json b/evalscope/evalscope/benchmarks/_meta/deep_swe.json new file mode 100644 index 0000000..c71c708 --- /dev/null +++ b/evalscope/evalscope/benchmarks/_meta/deep_swe.json @@ -0,0 +1,114 @@ +{ + "meta": { + "pretty_name": "DeepSWE", + "dataset_id": "evalscope/deep-swe", + "paper_url": null, + "tags": [ + "Coding", + "Agent", + "MultiTurn" + ], + "metrics": [ + "acc" + ], + "few_shot_num": 0, + "eval_split": "test", + "train_split": "", + "subset_list": [ + "default" + ], + "description": "\n## Overview\n\nDeepSWE is a coding-agent benchmark for evaluating repository-level software engineering tasks. EvalScope\nintegrates it through Pier and runs each benchmark sample as one Pier Python API job.\n\n## Task Description\n\n- **Task Type**: Agentic software engineering\n- **Input**: DeepSWE task directory containing task metadata and verifier assets\n- **Output**: A repository patch produced by a Pier built-in agent\n- **Scoring**: Binary verifier reward exposed as `acc`\n\n## Evaluation Notes\n\n- Requires **Python>=3.12**, Docker, and `pip install evalscope[deep_swe]`\n- Dataset defaults to ModelScope `evalscope/deep-swe`\n- DeepSWE runs through Pier's Docker environment in EvalScope\n- Use `pier_agent_kwargs={'model_class': 'litellm'}` for OpenAI-compatible providers that do not support Responses API\n", + "prompt_template": "{question}", + "system_prompt": "", + "few_shot_prompt_template": "", + "aggregation": "mean", + "extra_params": { + "task_ids": { + "type": "list", + "description": "Optional list of DeepSWE task ids to evaluate.", + "value": [] + }, + "languages": { + "type": "list", + "description": "Optional task language filter from manifest metadata.", + "value": [] + }, + "categories": { + "type": "list", + "description": "Optional task category filter from manifest metadata.", + "value": [] + }, + "sample_seed": { + "type": "int", + "description": "Optional deterministic shuffle seed applied before limit.", + "value": "" + }, + "pier_agent_kwargs": { + "type": "dict", + "description": "Extra kwargs passed to Pier AgentConfig.kwargs.", + "value": {} + } + }, + "sandbox_config": {}, + "category": "agent" + }, + "statistics": { + "total_samples": 113, + "subset_stats": [ + { + "name": "test", + "sample_count": 113, + "prompt_length_mean": 2158.07, + "prompt_length_min": 471, + "prompt_length_max": 5385, + "prompt_length_std": 1003.9, + "target_length_mean": null + } + ], + "prompt_length": { + "mean": 2158.07, + "min": 471, + "max": 5385, + "std": 1003.9 + }, + "target_length_mean": null, + "computed_at": "2026-07-06T17:32:56.616065" + }, + "sample_example": { + "data": { + "input": [ + { + "id": "f61040e0", + "content": "Add a new `errorStack` constructor option to SuperJSON. Omitting it leaves existing Error behavior unchanged.\n\nThe option shape is `{ mode?, normalizeNewlines?, trimLeadingWhitespace?, maxStackLines?, stripInternalFrames?, redactPaths?, inclu ... [TRUNCATED 3577 chars] ... ): Processor | undefined`. `normalizeErrorStackOptions` returns `undefined` for any non-object input (`null`, `undefined`, strings).\n\nBefore writing, read through the existing error serialization logic and the `allowedErrorProps` mechanism.\n\n" + } + ], + "target": "", + "id": 0, + "group_id": 0, + "metadata": { + "ext_id": "kh701jywhzgddknqwzsq6npjv98226tq", + "task_id": "superjson-error-stack-serialization", + "display_title": "Add error stack serialization to SuperJSON", + "display_description": "Add configurable serialization and restoration of error stacks, stack frames, causes, and sanitization in SuperJSON.", + "repo": "flightcontrolhq/superjson", + "repository_url": "https://github.com/flightcontrolhq/superjson.git", + "original_title": "Error Stack Serialization Support", + "category": "feature_request", + "language": "typescript", + "task_path": "~/.cache/evalscope/deep_swe/snapshots/evalscope/deep-swe/tasks/superjson-error-stack-serialization", + "task_toml_path": "~/.cache/evalscope/deep_swe/snapshots/evalscope/deep-swe/tasks/superjson-error-stack-serialization/task.toml", + "instruction": "Add a new `errorStack` constructor option to SuperJSON. Omitting it leaves existing Error behavior unchanged.\n\nThe option shape is `{ mode?, normalizeNewlines?, trimLeadingWhitespace?, maxStackLines?, stripInternalFrames?, redactPaths?, inclu ... [TRUNCATED 3577 chars] ... ): Processor | undefined`. `normalizeErrorStackOptions` returns `undefined` for any non-object input (`null`, `undefined`, strings).\n\nBefore writing, read through the existing error serialization logic and the `allowedErrorProps` mechanism.\n\n" + } + }, + "subset": "test", + "truncated": false + }, + "readme": { + "en": "# DeepSWE\n\n\n## Overview\n\nDeepSWE is a coding-agent benchmark for evaluating repository-level software engineering tasks. EvalScope\nintegrates it through Pier and runs each benchmark sample as one Pier Python API job.\n\n## Task Description\n\n- **Task Type**: Agentic software engineering\n- **Input**: DeepSWE task directory containing task metadata and verifier assets\n- **Output**: A repository patch produced by a Pier built-in agent\n- **Scoring**: Binary verifier reward exposed as `acc`\n\n## Evaluation Notes\n\n- Requires **Python>=3.12**, Docker, and `pip install evalscope[deep_swe]`\n- Dataset defaults to ModelScope `evalscope/deep-swe`\n- DeepSWE runs through Pier's Docker environment in EvalScope\n- Use `pier_agent_kwargs={'model_class': 'litellm'}` for OpenAI-compatible providers that do not support Responses API\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `deep_swe` |\n| **Dataset ID** | [evalscope/deep-swe](https://modelscope.cn/datasets/evalscope/deep-swe/summary) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `Coding`, `MultiTurn` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 113 |\n| Prompt Length (Mean) | 2158.07 chars |\n| Prompt Length (Min/Max) | 471 / 5385 chars |\n\n## Sample Example\n\n**Subset**: `test`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"f61040e0\",\n \"content\": \"Add a new `errorStack` constructor option to SuperJSON. Omitting it leaves existing Error behavior unchanged.\\n\\nThe option shape is `{ mode?, normalizeNewlines?, trimLeadingWhitespace?, maxStackLines?, stripInternalFrames?, redactPaths?, inclu ... [TRUNCATED 3577 chars] ... ): Processor | undefined`. `normalizeErrorStackOptions` returns `undefined` for any non-object input (`null`, `undefined`, strings).\\n\\nBefore writing, read through the existing error serialization logic and the `allowedErrorProps` mechanism.\\n\\n\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"ext_id\": \"kh701jywhzgddknqwzsq6npjv98226tq\",\n \"task_id\": \"superjson-error-stack-serialization\",\n \"display_title\": \"Add error stack serialization to SuperJSON\",\n \"display_description\": \"Add configurable serialization and restoration of error stacks, stack frames, causes, and sanitization in SuperJSON.\",\n \"repo\": \"flightcontrolhq/superjson\",\n \"repository_url\": \"https://github.com/flightcontrolhq/superjson.git\",\n \"original_title\": \"Error Stack Serialization Support\",\n \"category\": \"feature_request\",\n \"language\": \"typescript\",\n \"task_path\": \"~/.cache/evalscope/deep_swe/snapshots/evalscope/deep-swe/tasks/superjson-error-stack-serialization\",\n \"task_toml_path\": \"~/.cache/evalscope/deep_swe/snapshots/evalscope/deep-swe/tasks/superjson-error-stack-serialization/task.toml\",\n \"instruction\": \"Add a new `errorStack` constructor option to SuperJSON. Omitting it leaves existing Error behavior unchanged.\\n\\nThe option shape is `{ mode?, normalizeNewlines?, trimLeadingWhitespace?, maxStackLines?, stripInternalFrames?, redactPaths?, inclu ... [TRUNCATED 3577 chars] ... ): Processor | undefined`. `normalizeErrorStackOptions` returns `undefined` for any non-object input (`null`, `undefined`, strings).\\n\\nBefore writing, read through the existing error serialization logic and the `allowedErrorProps` mechanism.\\n\\n\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `task_ids` | `list` | `[]` | Optional list of DeepSWE task ids to evaluate. |\n| `languages` | `list` | `[]` | Optional task language filter from manifest metadata. |\n| `categories` | `list` | `[]` | Optional task category filter from manifest metadata. |\n| `sample_seed` | `int` | `` | Optional deterministic shuffle seed applied before limit. |\n| `pier_agent_kwargs` | `dict` | `{}` | Extra kwargs passed to Pier AgentConfig.kwargs. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets deep_swe \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['deep_swe'],\n dataset_args={\n 'deep_swe': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# DeepSWE\n\n\n## 概述\n\nDeepSWE 是一个用于评估仓库级软件工程任务的编码智能体基准测试。EvalScope 通过 Pier 集成该基准，并将每个基准样本作为一项 Pier Python API 任务运行。\n\n## 任务描述\n\n- **任务类型**：智能体软件工程\n- **输入**：包含任务元数据和验证器资源的 DeepSWE 任务目录\n- **输出**：由 Pier 内置智能体生成的代码仓库补丁\n- **评分方式**：二值验证器奖励，以 `acc` 形式暴露\n\n## 评估说明\n\n- 要求 **Python>=3.12**、Docker，以及执行 `pip install evalscope[deep_swe]`\n- 数据集默认使用 ModelScope 上的 `evalscope/deep-swe`\n- DeepSWE 在 EvalScope 中通过 Pier 的 Docker 环境运行\n- 对于不支持 Responses API 的 OpenAI 兼容提供商，请使用 `pier_agent_kwargs={'model_class': 'litellm'}`\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `deep_swe` |\n| **数据集ID** | [evalscope/deep-swe](https://modelscope.cn/datasets/evalscope/deep-swe/summary) |\n| **论文** | 无 |\n| **标签** | `Agent`, `Coding`, `MultiTurn` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 113 |\n| 提示词长度（平均） | 2158.07 字符 |\n| 提示词长度（最小/最大） | 471 / 5385 字符 |\n\n## 样例示例\n\n**子集**: `test`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"f61040e0\",\n \"content\": \"Add a new `errorStack` constructor option to SuperJSON. Omitting it leaves existing Error behavior unchanged.\\n\\nThe option shape is `{ mode?, normalizeNewlines?, trimLeadingWhitespace?, maxStackLines?, stripInternalFrames?, redactPaths?, inclu ... [TRUNCATED 3577 chars] ... ): Processor | undefined`. `normalizeErrorStackOptions` returns `undefined` for any non-object input (`null`, `undefined`, strings).\\n\\nBefore writing, read through the existing error serialization logic and the `allowedErrorProps` mechanism.\\n\\n\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"ext_id\": \"kh701jywhzgddknqwzsq6npjv98226tq\",\n \"task_id\": \"superjson-error-stack-serialization\",\n \"display_title\": \"Add error stack serialization to SuperJSON\",\n \"display_description\": \"Add configurable serialization and restoration of error stacks, stack frames, causes, and sanitization in SuperJSON.\",\n \"repo\": \"flightcontrolhq/superjson\",\n \"repository_url\": \"https://github.com/flightcontrolhq/superjson.git\",\n \"original_title\": \"Error Stack Serialization Support\",\n \"category\": \"feature_request\",\n \"language\": \"typescript\",\n \"task_path\": \"~/.cache/evalscope/deep_swe/snapshots/evalscope/deep-swe/tasks/superjson-error-stack-serialization\",\n \"task_toml_path\": \"~/.cache/evalscope/deep_swe/snapshots/evalscope/deep-swe/tasks/superjson-error-stack-serialization/task.toml\",\n \"instruction\": \"Add a new `errorStack` constructor option to SuperJSON. Omitting it leaves existing Error behavior unchanged.\\n\\nThe option shape is `{ mode?, normalizeNewlines?, trimLeadingWhitespace?, maxStackLines?, stripInternalFrames?, redactPaths?, inclu ... [TRUNCATED 3577 chars] ... ): Processor | undefined`. `normalizeErrorStackOptions` returns `undefined` for any non-object input (`null`, `undefined`, strings).\\n\\nBefore writing, read through the existing error serialization logic and the `allowedErrorProps` mechanism.\\n\\n\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板:**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `task_ids` | `list` | `[]` | 可选的 DeepSWE 任务 ID 列表，用于指定评估范围。 |\n| `languages` | `list` | `[]` | 可选的任务语言过滤器，基于清单元数据。 |\n| `categories` | `list` | `[]` | 可选的任务类别过滤器，基于清单元数据。 |\n| `sample_seed` | `int` | `` | 可选的确定性打乱种子，在限制样本数量前应用。 |\n| `pier_agent_kwargs` | `dict` | `{}` | 传递给 Pier AgentConfig.kwargs 的额外关键字参数。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets deep_swe \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['deep_swe'],\n dataset_args={\n 'deep_swe': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "54626e00b5f66f41f2353bba4826cd3c", + "needs_translation": false + }, + "updated_at": "2026-07-06T17:32:58.309507", + "translation_updated_at": "2026-07-06T17:33:00" +} diff --git a/evalscope/evalscope/benchmarks/_meta/docmath.json b/evalscope/evalscope/benchmarks/_meta/docmath.json index 02c301b..51326e3 100644 --- a/evalscope/evalscope/benchmarks/_meta/docmath.json +++ b/evalscope/evalscope/benchmarks/_meta/docmath.json @@ -105,4 +105,4 @@ }, "updated_at": "2026-01-28T17:31:32.213621", "translation_updated_at": "2026-01-28T17:21:59Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/docvqa.json b/evalscope/evalscope/benchmarks/_meta/docvqa.json index b66ce82..4d36b28 100644 --- a/evalscope/evalscope/benchmarks/_meta/docvqa.json +++ b/evalscope/evalscope/benchmarks/_meta/docvqa.json @@ -151,4 +151,4 @@ }, "updated_at": "2026-01-28T17:31:32.214406", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/drivel_binary.json b/evalscope/evalscope/benchmarks/_meta/drivel_binary.json index 145f622..d0fa7cf 100644 --- a/evalscope/evalscope/benchmarks/_meta/drivel_binary.json +++ b/evalscope/evalscope/benchmarks/_meta/drivel_binary.json @@ -80,4 +80,4 @@ }, "updated_at": "2026-01-28T17:31:32.222850", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/drivel_multilabel.json b/evalscope/evalscope/benchmarks/_meta/drivel_multilabel.json index 69de759..624ff1f 100644 --- a/evalscope/evalscope/benchmarks/_meta/drivel_multilabel.json +++ b/evalscope/evalscope/benchmarks/_meta/drivel_multilabel.json @@ -91,4 +91,4 @@ }, "updated_at": "2026-01-28T17:31:32.223704", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/drivel_selection.json b/evalscope/evalscope/benchmarks/_meta/drivel_selection.json index 92f56e4..441f5a1 100644 --- a/evalscope/evalscope/benchmarks/_meta/drivel_selection.json +++ b/evalscope/evalscope/benchmarks/_meta/drivel_selection.json @@ -87,4 +87,4 @@ }, "updated_at": "2026-01-28T17:31:32.225038", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/drivel_writing.json b/evalscope/evalscope/benchmarks/_meta/drivel_writing.json index bddd232..2f43d97 100644 --- a/evalscope/evalscope/benchmarks/_meta/drivel_writing.json +++ b/evalscope/evalscope/benchmarks/_meta/drivel_writing.json @@ -86,4 +86,4 @@ }, "updated_at": "2026-01-28T17:31:32.225015", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/drop.json b/evalscope/evalscope/benchmarks/_meta/drop.json index adc12c6..083abd9 100644 --- a/evalscope/evalscope/benchmarks/_meta/drop.json +++ b/evalscope/evalscope/benchmarks/_meta/drop.json @@ -120,4 +120,4 @@ }, "updated_at": "2026-01-28T17:31:32.227072", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/emb_spatial_bench.json b/evalscope/evalscope/benchmarks/_meta/emb_spatial_bench.json new file mode 100644 index 0000000..b7760bf --- /dev/null +++ b/evalscope/evalscope/benchmarks/_meta/emb_spatial_bench.json @@ -0,0 +1,312 @@ +{ + "meta": { + "pretty_name": "EmbSpatial-Bench", + "dataset_id": "evalscope/EmbSpatial-Bench", + "paper_url": "https://aclanthology.org/2024.acl-short.33/", + "tags": [ + "MultiModal", + "Reasoning", + "MCQ" + ], + "metrics": [ + "acc" + ], + "few_shot_num": 0, + "eval_split": "test", + "train_split": "", + "subset_list": [ + "close", + "far", + "above", + "under", + "left", + "right" + ], + "description": "\n## Overview\n\nEmbSpatial-Bench is a benchmark for evaluating embodied spatial understanding of large vision-language models (LVLMs). The benchmark is automatically derived from embodied scenes and covers 6 spatial relationships from an egocentric perspective: **close**, **far**, **above**, **under**, **left**, and **right**.\n\n## Task Description\n\n- **Task Type**: Multiple-Choice Visual Question Answering (VQA)\n- **Input**: An egocentric RGB image + a spatial reasoning question with 4 candidate answers\n- **Output**: A single letter (A / B / C / D) identifying the correct object or spatial relationship\n- **Domains**: Embodied AI, spatial reasoning (MP3D and AI2Thor environments)\n\n## Key Features\n\n- 3,640 human-verified evaluation questions derived from two embodied environments (MP3D and AI2Thor)\n- 6 spatial relation categories: close, far, above, under, left, right\n- Each question requires selecting the most spatially accurate answer from 4 options\n- Designed to expose the gap between current LVLMs and qualified embodied intelligence\n\n## Evaluation Notes\n\n- Default evaluation uses the **embspatial_bench.json** file (3,640 samples)\n- Primary metric: **Accuracy** (acc)\n- Answer indices are 0-based in the dataset (0 → A, 1 → B, 2 → C, 3 → D)\n- Images are stored as JPEG base64 strings in the JSON file\n- Subsets are organized by the `relation` field (6 spatial categories)\n- [Paper](https://aclanthology.org/2024.acl-short.33/) | [GitHub](https://github.com/mengfeidu/EmbSpatial-Bench)\n", + "prompt_template": "Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of {letters}. Think step by step before answering.\n\n{question}\n\n{choices}", + "system_prompt": "", + "few_shot_prompt_template": "", + "aggregation": "mean", + "extra_params": {}, + "sandbox_config": {}, + "category": "vlm" + }, + "statistics": { + "total_samples": 3640, + "subset_stats": [ + { + "name": "close", + "sample_count": 612, + "prompt_length_mean": 293.54, + "prompt_length_min": 274, + "prompt_length_max": 326, + "prompt_length_std": 10.6, + "target_length_mean": 1, + "multimodal": { + "has_images": true, + "has_audio": false, + "has_video": false, + "image": { + "count_total": 612, + "count_per_sample": { + "min": 1, + "max": 1, + "mean": 1 + }, + "resolutions": [ + "1296x968", + "300x300", + "640x480" + ], + "resolution_range": { + "min": "300x300", + "max": "1296x968" + }, + "formats": [ + "jpeg" + ] + } + } + }, + { + "name": "far", + "sample_count": 594, + "prompt_length_mean": 292.68, + "prompt_length_min": 265, + "prompt_length_max": 326, + "prompt_length_std": 12.55, + "target_length_mean": 1, + "multimodal": { + "has_images": true, + "has_audio": false, + "has_video": false, + "image": { + "count_total": 594, + "count_per_sample": { + "min": 1, + "max": 1, + "mean": 1 + }, + "resolutions": [ + "1296x968", + "300x300", + "640x480" + ], + "resolution_range": { + "min": "300x300", + "max": "1296x968" + }, + "formats": [ + "jpeg" + ] + } + } + }, + { + "name": "above", + "sample_count": 596, + "prompt_length_mean": 405.61, + "prompt_length_min": 360, + "prompt_length_max": 486, + "prompt_length_std": 23.28, + "target_length_mean": 1, + "multimodal": { + "has_images": true, + "has_audio": false, + "has_video": false, + "image": { + "count_total": 596, + "count_per_sample": { + "min": 1, + "max": 1, + "mean": 1 + }, + "resolutions": [ + "1296x968", + "300x300", + "640x480" + ], + "resolution_range": { + "min": "300x300", + "max": "1296x968" + }, + "formats": [ + "jpeg" + ] + } + } + }, + { + "name": "under", + "sample_count": 602, + "prompt_length_mean": 404.35, + "prompt_length_min": 350, + "prompt_length_max": 490, + "prompt_length_std": 22.27, + "target_length_mean": 1, + "multimodal": { + "has_images": true, + "has_audio": false, + "has_video": false, + "image": { + "count_total": 602, + "count_per_sample": { + "min": 1, + "max": 1, + "mean": 1 + }, + "resolutions": [ + "1296x968", + "300x300", + "640x480" + ], + "resolution_range": { + "min": "300x300", + "max": "1296x968" + }, + "formats": [ + "jpeg" + ] + } + } + }, + { + "name": "left", + "sample_count": 616, + "prompt_length_mean": 408.86, + "prompt_length_min": 359, + "prompt_length_max": 486, + "prompt_length_std": 21.53, + "target_length_mean": 1, + "multimodal": { + "has_images": true, + "has_audio": false, + "has_video": false, + "image": { + "count_total": 616, + "count_per_sample": { + "min": 1, + "max": 1, + "mean": 1 + }, + "resolutions": [ + "1296x968", + "300x300", + "640x480" + ], + "resolution_range": { + "min": "300x300", + "max": "1296x968" + }, + "formats": [ + "jpeg" + ] + } + } + }, + { + "name": "right", + "sample_count": 620, + "prompt_length_mean": 409.47, + "prompt_length_min": 352, + "prompt_length_max": 481, + "prompt_length_std": 21.76, + "target_length_mean": 1, + "multimodal": { + "has_images": true, + "has_audio": false, + "has_video": false, + "image": { + "count_total": 620, + "count_per_sample": { + "min": 1, + "max": 1, + "mean": 1 + }, + "resolutions": [ + "1296x968", + "300x300", + "640x480" + ], + "resolution_range": { + "min": "300x300", + "max": "1296x968" + }, + "formats": [ + "jpeg" + ] + } + } + } + ], + "prompt_length": { + "mean": 369.34, + "min": 265, + "max": 490, + "std": 57.07 + }, + "target_length_mean": 1, + "computed_at": "2026-07-06T17:23:58.082817", + "multimodal": { + "has_images": true, + "has_audio": false, + "has_video": false, + "image": { + "count_total": 3640, + "count_per_sample": { + "min": 1, + "max": 1, + "mean": 1 + }, + "resolutions": [ + "1296x968", + "300x300", + "640x480" + ], + "resolution_range": { + "min": "300x300", + "max": "1296x968" + }, + "formats": [ + "jpeg" + ] + } + } + }, + "sample_example": { + "data": { + "input": [ + { + "id": "3f406c29", + "content": [ + { + "image": "[BASE64_IMAGE: jpeg, ~35.2KB]" + }, + { + "text": "Among the listed objects, which one is closest to your current location in the image?\n(A) table\n(B) towel\n(C) door\n(D) basket\nAnswer with only the letter of the correct option. The last line of your response should be of the format: ANSWER: [LETTER] where LETTER is one of A, B, C, D." + } + ] + } + ], + "target": "D", + "id": 0, + "group_id": 0, + "subset_key": "close", + "metadata": { + "question_id": "mp3d_0", + "relation": "close", + "data_source": "mp3d" + } + }, + "subset": "close", + "truncated": false + }, + "readme": { + "en": "# EmbSpatial-Bench\n\n\n## Overview\n\nEmbSpatial-Bench is a benchmark for evaluating embodied spatial understanding of large vision-language models (LVLMs). The benchmark is automatically derived from embodied scenes and covers 6 spatial relationships from an egocentric perspective: **close**, **far**, **above**, **under**, **left**, and **right**.\n\n## Task Description\n\n- **Task Type**: Multiple-Choice Visual Question Answering (VQA)\n- **Input**: An egocentric RGB image + a spatial reasoning question with 4 candidate answers\n- **Output**: A single letter (A / B / C / D) identifying the correct object or spatial relationship\n- **Domains**: Embodied AI, spatial reasoning (MP3D and AI2Thor environments)\n\n## Key Features\n\n- 3,640 human-verified evaluation questions derived from two embodied environments (MP3D and AI2Thor)\n- 6 spatial relation categories: close, far, above, under, left, right\n- Each question requires selecting the most spatially accurate answer from 4 options\n- Designed to expose the gap between current LVLMs and qualified embodied intelligence\n\n## Evaluation Notes\n\n- Default evaluation uses the **embspatial_bench.json** file (3,640 samples)\n- Primary metric: **Accuracy** (acc)\n- Answer indices are 0-based in the dataset (0 → A, 1 → B, 2 → C, 3 → D)\n- Images are stored as JPEG base64 strings in the JSON file\n- Subsets are organized by the `relation` field (6 spatial categories)\n- [Paper](https://aclanthology.org/2024.acl-short.33/) | [GitHub](https://github.com/mengfeidu/EmbSpatial-Bench)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `emb_spatial_bench` |\n| **Dataset ID** | [evalscope/EmbSpatial-Bench](https://modelscope.cn/datasets/evalscope/EmbSpatial-Bench/summary) |\n| **Paper** | [Paper](https://aclanthology.org/2024.acl-short.33/) |\n| **Tags** | `MCQ`, `MultiModal`, `Reasoning` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 3,640 |\n| Prompt Length (Mean) | 369.34 chars |\n| Prompt Length (Min/Max) | 265 / 490 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `close` | 612 | 293.54 | 274 | 326 |\n| `far` | 594 | 292.68 | 265 | 326 |\n| `above` | 596 | 405.61 | 360 | 486 |\n| `under` | 602 | 404.35 | 350 | 490 |\n| `left` | 616 | 408.86 | 359 | 486 |\n| `right` | 620 | 409.47 | 352 | 481 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 3,640 |\n| Images per Sample | min: 1, max: 1, mean: 1 |\n| Resolution Range | 300x300 - 1296x968 |\n| Formats | jpeg |\n\n\n## Sample Example\n\n**Subset**: `close`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"3f406c29\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~35.2KB]\"\n },\n {\n \"text\": \"Among the listed objects, which one is closest to your current location in the image?\\n(A) table\\n(B) towel\\n(C) door\\n(D) basket\\nAnswer with only the letter of the correct option. The last line of your response should be of the format: ANSWER: [LETTER] where LETTER is one of A, B, C, D.\"\n }\n ]\n }\n ],\n \"target\": \"D\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"close\",\n \"metadata\": {\n \"question_id\": \"mp3d_0\",\n \"relation\": \"close\",\n \"data_source\": \"mp3d\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\nAnswer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of {letters}. Think step by step before answering.\n\n{question}\n\n{choices}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets emb_spatial_bench \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['emb_spatial_bench'],\n dataset_args={\n 'emb_spatial_bench': {\n # subset_list: ['close', 'far', 'above'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# EmbSpatial-Bench\n\n\n## 概述\n\nEmbSpatial-Bench 是一个用于评估大视觉语言模型（LVLMs）具身空间理解能力的基准测试。该基准测试从具身场景中自动构建，涵盖从第一人称视角出发的 6 种空间关系：**close**（近）、**far**（远）、**above**（上）、**under**（下）、**left**（左）和 **right**（右）。\n\n## 任务描述\n\n- **任务类型**：多项选择视觉问答（VQA）\n- **输入**：一张第一人称 RGB 图像 + 一个包含 4 个候选答案的空间推理问题\n- **输出**：单个字母（A / B / C / D），标识正确的物体或空间关系\n- **领域**：具身人工智能、空间推理（MP3D 和 AI2Thor 环境）\n\n## 主要特点\n\n- 包含 3,640 个人工验证的评测问题，源自两个具身环境（MP3D 和 AI2Thor）\n- 涵盖 6 类空间关系：close、far、above、under、left、right\n- 每个问题需从 4 个选项中选出空间上最准确的答案\n- 旨在揭示当前 LVLM 与合格具身智能之间的差距\n\n## 评测说明\n\n- 默认评测使用 **embspatial_bench.json** 文件（共 3,640 个样本）\n- 主要指标：**准确率**（acc）\n- 数据集中答案索引为 0 起始（0 → A，1 → B，2 → C，3 → D）\n- 图像以 JPEG base64 字符串形式存储在 JSON 文件中\n- 子集按 `relation` 字段组织（6 种空间关系类别）\n- [论文](https://aclanthology.org/2024.acl-short.33/) | [GitHub](https://github.com/mengfeidu/EmbSpatial-Bench)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `emb_spatial_bench` |\n| **数据集ID** | [evalscope/EmbSpatial-Bench](https://modelscope.cn/datasets/evalscope/EmbSpatial-Bench/summary) |\n| **论文** | [Paper](https://aclanthology.org/2024.acl-short.33/) |\n| **标签** | `MCQ`, `MultiModal`, `Reasoning` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评测划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 3,640 |\n| 提示词长度（平均） | 369.34 字符 |\n| 提示词长度（最小/最大） | 265 / 490 字符 |\n\n**各子集统计信息：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `close` | 612 | 293.54 | 274 | 326 |\n| `far` | 594 | 292.68 | 265 | 326 |\n| `above` | 596 | 405.61 | 360 | 486 |\n| `under` | 602 | 404.35 | 350 | 490 |\n| `left` | 616 | 408.86 | 359 | 486 |\n| `right` | 620 | 409.47 | 352 | 481 |\n\n**图像统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 3,640 |\n| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 |\n| 分辨率范围 | 300x300 - 1296x968 |\n| 格式 | jpeg |\n\n\n## 样例示例\n\n**子集**: `close`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"3f406c29\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~35.2KB]\"\n },\n {\n \"text\": \"Among the listed objects, which one is closest to your current location in the image?\\n(A) table\\n(B) towel\\n(C) door\\n(D) basket\\nAnswer with only the letter of the correct option. The last line of your response should be of the format: ANSWER: [LETTER] where LETTER is one of A, B, C, D.\"\n }\n ]\n }\n ],\n \"target\": \"D\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"close\",\n \"metadata\": {\n \"question_id\": \"mp3d_0\",\n \"relation\": \"close\",\n \"data_source\": \"mp3d\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\nAnswer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of {letters}. Think step by step before answering.\n\n{question}\n\n{choices}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets emb_spatial_bench \\\n --limit 10 # 正式评测时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['emb_spatial_bench'],\n dataset_args={\n 'emb_spatial_bench': {\n # subset_list: ['close', 'far', 'above'] # 可选，用于评测特定子集\n }\n },\n limit=10, # 正式评测时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "1a472c56595a22ddb88cd289b3a34111", + "needs_translation": false + }, + "updated_at": "2026-07-06T17:58:15.970262", + "translation_updated_at": "2026-07-06T17:58:37" +} diff --git a/evalscope/evalscope/benchmarks/_meta/eq_bench.json b/evalscope/evalscope/benchmarks/_meta/eq_bench.json index f0f9161..4904bae 100644 --- a/evalscope/evalscope/benchmarks/_meta/eq_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/eq_bench.json @@ -91,4 +91,4 @@ }, "updated_at": "2026-01-28T17:31:32.228078", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/erqa.json b/evalscope/evalscope/benchmarks/_meta/erqa.json index 6bb45e3..217b80e 100644 --- a/evalscope/evalscope/benchmarks/_meta/erqa.json +++ b/evalscope/evalscope/benchmarks/_meta/erqa.json @@ -442,4 +442,4 @@ }, "updated_at": "2026-07-02T19:24:50.476503", "translation_updated_at": "2026-07-02T19:26:43" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/evalmuse.json b/evalscope/evalscope/benchmarks/_meta/evalmuse.json index bfa0a6f..6fcc1c2 100644 --- a/evalscope/evalscope/benchmarks/_meta/evalmuse.json +++ b/evalscope/evalscope/benchmarks/_meta/evalmuse.json @@ -98,4 +98,4 @@ }, "updated_at": "2026-01-28T17:31:32.506303", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/fin_ner.json b/evalscope/evalscope/benchmarks/_meta/fin_ner.json index c5dcb2a..215b2f9 100644 --- a/evalscope/evalscope/benchmarks/_meta/fin_ner.json +++ b/evalscope/evalscope/benchmarks/_meta/fin_ner.json @@ -147,4 +147,4 @@ }, "updated_at": "2026-01-28T17:31:32.356575", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/fleurs.json b/evalscope/evalscope/benchmarks/_meta/fleurs.json index 59cbe6b..d33097d 100644 --- a/evalscope/evalscope/benchmarks/_meta/fleurs.json +++ b/evalscope/evalscope/benchmarks/_meta/fleurs.json @@ -180,4 +180,4 @@ }, "updated_at": "2026-01-28T17:31:32.230215", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/frames.json b/evalscope/evalscope/benchmarks/_meta/frames.json index 65918d9..eb64605 100644 --- a/evalscope/evalscope/benchmarks/_meta/frames.json +++ b/evalscope/evalscope/benchmarks/_meta/frames.json @@ -110,4 +110,4 @@ }, "updated_at": "2026-01-28T17:31:32.231481", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/gaia.json b/evalscope/evalscope/benchmarks/_meta/gaia.json index a871c79..6c6b26a 100644 --- a/evalscope/evalscope/benchmarks/_meta/gaia.json +++ b/evalscope/evalscope/benchmarks/_meta/gaia.json @@ -19,34 +19,17 @@ "2023_level2", "2023_level3" ], - "description": "\n## Overview\n\nGAIA (General AI Assistants) is a benchmark of 450+ questions targeting next-generation LLMs with tool use, web browsing and multi-step reasoning. Each question has an unambiguous short answer and is bucketed into one of three difficulty levels.\n\n## Task Description\n\n- **Task Type**: Tool-use Agent (multi-turn)\n- **Input**: Natural-language question, optionally with one referenced attachment file (PDF / xlsx / image / audio / ...)\n- **Output**: A short final answer (number / short phrase / comma-separated list)\n- **Splits**: ``validation`` (answers public) and ``test`` (answers private — evaluation skipped)\n\n## Key Features\n\n- ReAct agent loop with a single ``bash`` tool inside a Docker sandbox (default image ``python:3.11``, includes ``curl`` / ``wget`` / ``git``).\n- Attachment files are mounted read-only at ``/shared_files`` inside the sandbox.\n- Rule-based scorer ported verbatim from the official GAIA leaderboard (no LLM judge).\n- Dataset downloaded from ModelScope (``gaia-benchmark/GAIA``) by default; set ``dataset_hub='huggingface'`` to load from Hugging Face instead.\n\n## Evaluation Notes\n\n- Requires Docker daemon running locally (or a remote sandbox engine via the ms_enclave configuration).\n- ``extra_params.max_steps`` caps the agent loop length (default 50).\n- ``extra_params.command_timeout`` sets per-``bash`` command timeout (default 180s, mirrors inspect_ai).\n- Network is enabled by default — many questions require browsing/searching.\n- Use ``subset_list`` to restrict to specific difficulty levels, e.g. ``['2023_level1']``, ``['2023_level1', '2023_level2']`` or ``['2023_all']`` (default).\n- [Usage Documentation](https://evalscope.readthedocs.io/en/latest/third_party/gaia.html)\n", + "description": "\n## Overview\n\nGAIA (General AI Assistants) is a benchmark of 450+ questions targeting next-generation LLMs with tool use, web browsing and multi-step reasoning. Each question has an unambiguous short answer and is bucketed into one of three difficulty levels.\n\n## Task Description\n\n- **Task Type**: Tool-use Agent (multi-turn)\n- **Input**: Natural-language question, optionally with one referenced attachment file (PDF / xlsx / image / audio / ...)\n- **Output**: A short final answer (number / short phrase / comma-separated list)\n- **Splits**: ``validation`` (answers public) and ``test`` (answers private — evaluation skipped)\n\n## Key Features\n\n- ReAct agent loop with a single ``bash`` tool inside a Docker sandbox (default image ``python:3.11``, includes ``curl`` / ``wget`` / ``git``).\n- Attachment files are mounted read-only at ``/shared_files`` inside the sandbox.\n- Rule-based scorer ported verbatim from the official GAIA leaderboard (no LLM judge).\n- Dataset downloaded from ModelScope (``gaia-benchmark/GAIA``) by default; set ``dataset_hub='huggingface'`` to load from Hugging Face instead.\n\n## Evaluation Notes\n\n- Requires Docker daemon running locally (or a remote sandbox engine via the ms_enclave configuration).\n- The agent loop defaults to 50 steps. Use ``NativeAgentConfig.max_steps`` to override it.\n- Network is enabled by default because many questions require browsing. Override the image, network, CPU or memory\n settings through ``TaskConfig.sandbox.default_config``.\n- Use ``subset_list`` to restrict to specific difficulty levels, e.g. ``['2023_level1']``, ``['2023_level1', '2023_level2']`` or ``['2023_all']`` (default).\n- [Usage Documentation](https://evalscope.readthedocs.io/en/latest/third_party/gaia.html)\n", "prompt_template": "{question}", "system_prompt": "", "few_shot_prompt_template": "", "aggregation": "mean", - "extra_params": { - "max_steps": { - "type": "int", - "description": "Maximum number of agent steps per sample.", - "value": 50 - }, - "command_timeout": { - "type": "float", - "description": "Default per-bash-command timeout in seconds.", - "value": 180.0 - }, - "docker_image": { - "type": "str", - "description": "Docker image used as the per-sample sandbox.", - "value": "python:3.11" - }, - "network_enabled": { - "type": "bool", - "description": "Allow the sandbox to access the network (GAIA browsing questions need this).", - "value": true - } - }, + "extra_params": {}, "sandbox_config": {}, + "agent_config": { + "strategy": "react", + "max_steps": 50 + }, "category": "agent" }, "statistics": { @@ -140,11 +123,11 @@ "truncated": false }, "readme": { - "en": "# GAIA\n\n\n## Overview\n\nGAIA (General AI Assistants) is a benchmark of 450+ questions targeting next-generation LLMs with tool use, web browsing and multi-step reasoning. Each question has an unambiguous short answer and is bucketed into one of three difficulty levels.\n\n## Task Description\n\n- **Task Type**: Tool-use Agent (multi-turn)\n- **Input**: Natural-language question, optionally with one referenced attachment file (PDF / xlsx / image / audio / ...)\n- **Output**: A short final answer (number / short phrase / comma-separated list)\n- **Splits**: ``validation`` (answers public) and ``test`` (answers private — evaluation skipped)\n\n## Key Features\n\n- ReAct agent loop with a single ``bash`` tool inside a Docker sandbox (default image ``python:3.11``, includes ``curl`` / ``wget`` / ``git``).\n- Attachment files are mounted read-only at ``/shared_files`` inside the sandbox.\n- Rule-based scorer ported verbatim from the official GAIA leaderboard (no LLM judge).\n- Dataset downloaded from ModelScope (``gaia-benchmark/GAIA``) by default; set ``dataset_hub='huggingface'`` to load from Hugging Face instead.\n\n## Evaluation Notes\n\n- Requires Docker daemon running locally (or a remote sandbox engine via the ms_enclave configuration).\n- ``extra_params.max_steps`` caps the agent loop length (default 50).\n- ``extra_params.command_timeout`` sets per-``bash`` command timeout (default 180s, mirrors inspect_ai).\n- Network is enabled by default — many questions require browsing/searching.\n- Use ``subset_list`` to restrict to specific difficulty levels, e.g. ``['2023_level1']``, ``['2023_level1', '2023_level2']`` or ``['2023_all']`` (default).\n- [Usage Documentation](https://evalscope.readthedocs.io/en/latest/third_party/gaia.html)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `gaia` |\n| **Dataset ID** | [gaia-benchmark/GAIA](https://modelscope.cn/datasets/gaia-benchmark/GAIA/summary) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `MultiTurn`, `Reasoning` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `validation` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 165 |\n| Prompt Length (Mean) | 861.35 chars |\n| Prompt Length (Min/Max) | 596 / 2582 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `2023_level1` | 53 | 906.53 | 604 | 2582 |\n| `2023_level2` | 86 | 816.66 | 596 | 1275 |\n| `2023_level3` | 26 | 917.08 | 621 | 1497 |\n\n## Sample Example\n\n**Subset**: `2023_level1`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"93e8257c\",\n \"content\": \"Please answer the question below. You should:\\n\\n- Return only your answer, which should be a number, or a short phrase with as few words as possible, or a comma separated list of numbers and/or strings.\\n- If the answer is a number, return only ... [TRUNCATED 431 chars] ... Earth and the Moon its closest approach? Please use the minimum perigee value on the Wikipedia page for the Moon when carrying out your calculation. Round your result to the nearest 1000 hours and do not use any comma separators if necessary.\"\n }\n ],\n \"target\": \"17\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"task_id\": \"e1fc63a2-da7a-432f-be78-7c4a95598703\",\n \"level\": \"1\",\n \"file_name\": \"\",\n \"file_path\": \"\",\n \"Annotator Metadata\": {\n \"Steps\": \"1. Googled Eliud Kipchoge marathon pace to find 4min 37sec/mile\\n2. Converted into fractions of hours.\\n3. Found moon periapsis in miles (225,623 miles).\\n4. Multiplied the two to find the number of hours and rounded to the nearest 100 hours.\",\n \"Number of steps\": \"4\",\n \"How long did this take?\": \"20 Minutes\",\n \"Tools\": \"1. A web browser.\\n2. A search engine.\\n3. A calculator.\",\n \"Number of tools\": \"3\"\n }\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `max_steps` | `int` | `50` | Maximum number of agent steps per sample. |\n| `command_timeout` | `float` | `180.0` | Default per-bash-command timeout in seconds. |\n| `docker_image` | `str` | `python:3.11` | Docker image used as the per-sample sandbox. |\n| `network_enabled` | `bool` | `True` | Allow the sandbox to access the network (GAIA browsing questions need this). |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets gaia \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['gaia'],\n dataset_args={\n 'gaia': {\n # subset_list: ['2023_level1', '2023_level2', '2023_level3'] # optional, evaluate specific subsets\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# GAIA\n\n\n## 概述\n\nGAIA（General AI Assistants）是一个包含 450 多个问题的基准测试，旨在评估具备工具使用、网页浏览和多步推理能力的新一代大语言模型（LLM）。每个问题都有一个明确的简短答案，并被划分为三个难度等级之一。\n\n## 任务描述\n\n- **任务类型**：工具使用型智能体（多轮交互）\n- **输入**：自然语言问题，可选附带一个引用附件文件（PDF / xlsx / 图像 / 音频 / ...）\n- **输出**：一个简短的最终答案（数字 / 短语 / 逗号分隔的列表）\n- **数据划分**：``validation``（答案公开）和 ``test``（答案私有 — 跳过评估）\n\n## 核心特性\n\n- 基于 ReAct 智能体循环，内置单个 ``bash`` 工具，运行于 Docker 沙箱中（默认镜像为 ``python:3.11``，已预装 ``curl`` / ``wget`` / ``git``）。\n- 附件文件以只读方式挂载至沙箱内的 ``/shared_files`` 目录。\n- 评分器直接移植自官方 GAIA 排行榜的规则逻辑（不使用 LLM 作为评判）。\n- 默认从 ModelScope 下载数据集（``gaia-benchmark/GAIA``）；设置 ``dataset_hub='huggingface'`` 可改为从 Hugging Face 加载。\n\n## 评估说明\n\n- 需要在本地运行 Docker 守护进程（或通过 ms_enclave 配置使用远程沙箱引擎）。\n- ``extra_params.max_steps`` 限制智能体循环的最大步数（默认为 50）。\n- ``extra_params.command_timeout`` 设置每条 ``bash`` 命令的超时时间（默认 180 秒，与 inspect_ai 一致）。\n- 默认启用网络访问 — 许多问题需要进行网页浏览或搜索。\n- 使用 ``subset_list`` 可限定特定难度级别，例如 ``['2023_level1']``、``['2023_level1', '2023_level2']`` 或默认的 ``['2023_all']``。\n- [使用文档](https://evalscope.readthedocs.io/zh-cn/latest/third_party/gaia.html)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `gaia` |\n| **数据集ID** | [gaia-benchmark/GAIA](https://modelscope.cn/datasets/gaia-benchmark/GAIA/summary) |\n| **论文** | N/A |\n| **标签** | `Agent`, `MultiTurn`, `Reasoning` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `validation` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 165 |\n| 提示词长度（平均） | 861.35 字符 |\n| 提示词长度（最小/最大） | 596 / 2582 字符 |\n\n**各子集统计信息：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `2023_level1` | 53 | 906.53 | 604 | 2582 |\n| `2023_level2` | 86 | 816.66 | 596 | 1275 |\n| `2023_level3` | 26 | 917.08 | 621 | 1497 |\n\n## 样例示例\n\n**子集**: `2023_level1`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"93e8257c\",\n \"content\": \"Please answer the question below. You should:\\n\\n- Return only your answer, which should be a number, or a short phrase with as few words as possible, or a comma separated list of numbers and/or strings.\\n- If the answer is a number, return only ... [TRUNCATED 431 chars] ... Earth and the Moon its closest approach? Please use the minimum perigee value on the Wikipedia page for the Moon when carrying out your calculation. Round your result to the nearest 1000 hours and do not use any comma separators if necessary.\"\n }\n ],\n \"target\": \"17\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"task_id\": \"e1fc63a2-da7a-432f-be78-7c4a95598703\",\n \"level\": \"1\",\n \"file_name\": \"\",\n \"file_path\": \"\",\n \"Annotator Metadata\": {\n \"Steps\": \"1. Googled Eliud Kipchoge marathon pace to find 4min 37sec/mile\\n2. Converted into fractions of hours.\\n3. Found moon periapsis in miles (225,623 miles).\\n4. Multiplied the two to find the number of hours and rounded to the nearest 100 hours.\",\n \"Number of steps\": \"4\",\n \"How long did this take?\": \"20 Minutes\",\n \"Tools\": \"1. A web browser.\\n2. A search engine.\\n3. A calculator.\",\n \"Number of tools\": \"3\"\n }\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `max_steps` | `int` | `50` | 每个样本允许的最大智能体步数。 |\n| `command_timeout` | `float` | `180.0` | 每条 bash 命令的默认超时时间（秒）。 |\n| `docker_image` | `str` | `python:3.11` | 用作每个样本沙箱环境的 Docker 镜像。 |\n| `network_enabled` | `bool` | `True` | 是否允许沙箱访问网络（GAIA 中涉及浏览的问题需要此选项）。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets gaia \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['gaia'],\n dataset_args={\n 'gaia': {\n # subset_list: ['2023_level1', '2023_level2', '2023_level3'] # 可选，用于评估特定子集\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "628ec8e67277a0ae28b0f6a8a6a4d09e", + "en": "# GAIA\n\n\n## Overview\n\nGAIA (General AI Assistants) is a benchmark of 450+ questions targeting next-generation LLMs with tool use, web browsing and multi-step reasoning. Each question has an unambiguous short answer and is bucketed into one of three difficulty levels.\n\n## Task Description\n\n- **Task Type**: Tool-use Agent (multi-turn)\n- **Input**: Natural-language question, optionally with one referenced attachment file (PDF / xlsx / image / audio / ...)\n- **Output**: A short final answer (number / short phrase / comma-separated list)\n- **Splits**: ``validation`` (answers public) and ``test`` (answers private — evaluation skipped)\n\n## Key Features\n\n- ReAct agent loop with a single ``bash`` tool inside a Docker sandbox (default image ``python:3.11``, includes ``curl`` / ``wget`` / ``git``).\n- Attachment files are mounted read-only at ``/shared_files`` inside the sandbox.\n- Rule-based scorer ported verbatim from the official GAIA leaderboard (no LLM judge).\n- Dataset downloaded from ModelScope (``gaia-benchmark/GAIA``) by default; set ``dataset_hub='huggingface'`` to load from Hugging Face instead.\n\n## Evaluation Notes\n\n- Requires Docker daemon running locally (or a remote sandbox engine via the ms_enclave configuration).\n- The agent loop defaults to 50 steps. Use ``NativeAgentConfig.max_steps`` to override it.\n- Network is enabled by default because many questions require browsing. Override the image, network, CPU or memory\n settings through ``TaskConfig.sandbox.default_config``.\n- Use ``subset_list`` to restrict to specific difficulty levels, e.g. ``['2023_level1']``, ``['2023_level1', '2023_level2']`` or ``['2023_all']`` (default).\n- [Usage Documentation](https://evalscope.readthedocs.io/en/latest/third_party/gaia.html)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `gaia` |\n| **Dataset ID** | [gaia-benchmark/GAIA](https://modelscope.cn/datasets/gaia-benchmark/GAIA/summary) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `MultiTurn`, `Reasoning` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `validation` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 165 |\n| Prompt Length (Mean) | 861.35 chars |\n| Prompt Length (Min/Max) | 596 / 2582 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `2023_level1` | 53 | 906.53 | 604 | 2582 |\n| `2023_level2` | 86 | 816.66 | 596 | 1275 |\n| `2023_level3` | 26 | 917.08 | 621 | 1497 |\n\n## Sample Example\n\n**Subset**: `2023_level1`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"93e8257c\",\n \"content\": \"Please answer the question below. You should:\\n\\n- Return only your answer, which should be a number, or a short phrase with as few words as possible, or a comma separated list of numbers and/or strings.\\n- If the answer is a number, return only ... [TRUNCATED 431 chars] ... Earth and the Moon its closest approach? Please use the minimum perigee value on the Wikipedia page for the Moon when carrying out your calculation. Round your result to the nearest 1000 hours and do not use any comma separators if necessary.\"\n }\n ],\n \"target\": \"17\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"task_id\": \"e1fc63a2-da7a-432f-be78-7c4a95598703\",\n \"level\": \"1\",\n \"file_name\": \"\",\n \"file_path\": \"\",\n \"Annotator Metadata\": {\n \"Steps\": \"1. Googled Eliud Kipchoge marathon pace to find 4min 37sec/mile\\n2. Converted into fractions of hours.\\n3. Found moon periapsis in miles (225,623 miles).\\n4. Multiplied the two to find the number of hours and rounded to the nearest 100 hours.\",\n \"Number of steps\": \"4\",\n \"How long did this take?\": \"20 Minutes\",\n \"Tools\": \"1. A web browser.\\n2. A search engine.\\n3. A calculator.\",\n \"Number of tools\": \"3\"\n }\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets gaia \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"react\",\"max_steps\":50}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['gaia'],\n agent_config=NativeAgentConfig(\n strategy='react',\n max_steps=50,\n ),\n dataset_args={\n 'gaia': {\n # subset_list: ['2023_level1', '2023_level2', '2023_level3'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# GAIA\n\n\n## 概述\n\nGAIA（General AI Assistants）是一个包含 450 多个问题的基准测试，旨在评估具备工具使用、网页浏览和多步推理能力的新一代大语言模型（LLM）。每个问题都有一个明确的简短答案，并被划分为三个难度等级之一。\n\n## 任务描述\n\n- **任务类型**：工具使用型智能体（多轮交互）\n- **输入**：自然语言问题，可选附带一个引用附件文件（PDF / xlsx / 图像 / 音频 / ...）\n- **输出**：简短的最终答案（数字 / 短语 / 逗号分隔的列表）\n- **数据划分**：``validation``（答案公开）和 ``test``（答案私有 — 跳过评估）\n\n## 核心特性\n\n- 基于 ReAct 智能体循环，内置单一 ``bash`` 工具，运行于 Docker 沙箱中（默认镜像为 ``python:3.11``，已预装 ``curl`` / ``wget`` / ``git``）。\n- 附件文件以只读方式挂载至沙箱内的 ``/shared_files`` 目录。\n- 评分器直接移植自官方 GAIA 排行榜的规则逻辑（不使用 LLM 作为评判）。\n- 默认从 ModelScope 下载数据集（``gaia-benchmark/GAIA``）；设置 ``dataset_hub='huggingface'`` 可改为从 Hugging Face 加载。\n\n## 评估说明\n\n- 需要在本地运行 Docker 守护进程（或通过 ms_enclave 配置使用远程沙箱引擎）。\n- 智能体循环默认最多执行 50 步。可通过 ``NativeAgentConfig.max_steps`` 覆盖该设置。\n- 默认启用网络访问，因为许多问题需要网页浏览。可通过 ``TaskConfig.sandbox.default_config`` 覆盖镜像、网络、CPU 或内存等配置。\n- 使用 ``subset_list`` 限制评估特定难度级别，例如 ``['2023_level1']``、``['2023_level1', '2023_level2']`` 或默认的 ``['2023_all']``。\n- [使用文档](https://evalscope.readthedocs.io/zh-cn/latest/third_party/gaia.html)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `gaia` |\n| **数据集ID** | [gaia-benchmark/GAIA](https://modelscope.cn/datasets/gaia-benchmark/GAIA/summary) |\n| **论文** | N/A |\n| **标签** | `Agent`, `MultiTurn`, `Reasoning` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `validation` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 165 |\n| 提示词长度（平均） | 861.35 字符 |\n| 提示词长度（最小/最大） | 596 / 2582 字符 |\n\n**各子集统计数据：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `2023_level1` | 53 | 906.53 | 604 | 2582 |\n| `2023_level2` | 86 | 816.66 | 596 | 1275 |\n| `2023_level3` | 26 | 917.08 | 621 | 1497 |\n\n## 样例示例\n\n**子集**: `2023_level1`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"93e8257c\",\n \"content\": \"Please answer the question below. You should:\\n\\n- Return only your answer, which should be a number, or a short phrase with as few words as possible, or a comma separated list of numbers and/or strings.\\n- If the answer is a number, return only ... [TRUNCATED 431 chars] ... Earth and the Moon its closest approach? Please use the minimum perigee value on the Wikipedia page for the Moon when carrying out your calculation. Round your result to the nearest 1000 hours and do not use any comma separators if necessary.\"\n }\n ],\n \"target\": \"17\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"task_id\": \"e1fc63a2-da7a-432f-be78-7c4a95598703\",\n \"level\": \"1\",\n \"file_name\": \"\",\n \"file_path\": \"\",\n \"Annotator Metadata\": {\n \"Steps\": \"1. Googled Eliud Kipchoge marathon pace to find 4min 37sec/mile\\n2. Converted into fractions of hours.\\n3. Found moon periapsis in miles (225,623 miles).\\n4. Multiplied the two to find the number of hours and rounded to the nearest 100 hours.\",\n \"Number of steps\": \"4\",\n \"How long did this take?\": \"20 Minutes\",\n \"Tools\": \"1. A web browser.\\n2. A search engine.\\n3. A calculator.\",\n \"Number of tools\": \"3\"\n }\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets gaia \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"react\",\"max_steps\":50}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['gaia'],\n agent_config=NativeAgentConfig(\n strategy='react',\n max_steps=50,\n ),\n dataset_args={\n 'gaia': {\n # subset_list: ['2023_level1', '2023_level2', '2023_level3'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "fa1a747373905ae3236671850d639fe4", "needs_translation": false }, - "updated_at": "2026-06-01T14:08:06.294860", - "translation_updated_at": "2026-06-01T14:08:20" -} \ No newline at end of file + "updated_at": "2026-07-10T20:24:34.552553", + "translation_updated_at": "2026-07-10T20:24:42" +} diff --git a/evalscope/evalscope/benchmarks/_meta/gdpval.json b/evalscope/evalscope/benchmarks/_meta/gdpval.json index 8b187cb..4471e83 100644 --- a/evalscope/evalscope/benchmarks/_meta/gdpval.json +++ b/evalscope/evalscope/benchmarks/_meta/gdpval.json @@ -17,37 +17,17 @@ "subset_list": [ "default" ], - "description": "\n## Overview\n\nGDPval evaluates whether models can complete realistic economically valuable work tasks and produce requested\ndeliverable files. This adapter targets OpenAI's public 220-task gold subset mirrored on ModelScope as\n`openai-mirror/gdpval`.\n\n## Task Description\n\n- **Task Type**: Agentic professional work / deliverable generation\n- **Input**: A workplace-style task prompt, optionally with reference files\n- **Output**: Final response text and requested files under `deliverable_files/`\n- **Dataset**: OpenAI public GDPval gold subset with 220 tasks\n\n## Key Features\n\n- Uses the native EvalScope `AgentLoopAdapter` with bash and Python execution tools.\n- Loads records and reference files from ModelScope by default.\n- Mounts selected reference files read-only into the sandbox under `/reference_files`.\n- Extracts files written to `deliverable_files/` before sandbox teardown.\n- Generates a GDPval submission package with `deliverable_text` and `deliverable_files` columns.\n\n## Evaluation Notes\n\n- The default Docker image is `evalscope/gdpval:latest` and is built automatically from the bundled Dockerfile when\n missing. Set `extra_params.auto_build_docker_image=false` to require a pre-built image, or override\n `extra_params.docker_image`.\n- `submission_ready` is a local readiness metric: it is 1 when the model produced final text or at least one\n deliverable file. It is not an official GDPval quality score.\n- EvalScope does not run a local GDPval judge. Use the exported submission package with OpenAI's official GDPval judge\n to obtain quality scores.\n- Full document/spreadsheet/slide quality depends on the GDPval runtime image. Thin Python images are useful only for\n plumbing smoke tests.\n\n## Scoring and Submission\n\n- EvalScope writes a local submission folder under the reports directory.\n- The submission contains `deliverable_text` and `deliverable_files` fields in the GDPval dataset format.\n- Official GDPval grading is external. Run OpenAI's official GDPval judge on the exported submission package.\n", + "description": "\n## Overview\n\nGDPval evaluates whether models can complete realistic economically valuable work tasks and produce requested\ndeliverable files. This adapter targets OpenAI's public 220-task gold subset mirrored on ModelScope as\n`openai-mirror/gdpval`.\n\n## Task Description\n\n- **Task Type**: Agentic professional work / deliverable generation\n- **Input**: A workplace-style task prompt, optionally with reference files\n- **Output**: Final response text and requested files under `deliverable_files/`\n- **Dataset**: OpenAI public GDPval gold subset with 220 tasks\n\n## Key Features\n\n- Uses the native EvalScope `AgentLoopAdapter` with bash and Python execution tools.\n- Loads records and reference files from ModelScope by default.\n- Mounts selected reference files read-only into the sandbox under `/reference_files`.\n- Extracts files written to `deliverable_files/` before sandbox teardown.\n- Generates a GDPval submission package with `deliverable_text` and `deliverable_files` columns.\n\n## Evaluation Notes\n\n- The default Docker image is built automatically from the bundled Dockerfile into a content-hashed local tag. Set\n `extra_params.auto_build_docker_image=false` to require a pre-built `evalscope/gdpval:latest`. Override the image,\n network, CPU or memory settings through `TaskConfig.sandbox.default_config`.\n- `submission_ready` is a local readiness metric: it is 1 when the model produced final text or at least one\n deliverable file. It is not an official GDPval quality score.\n- EvalScope does not run a local GDPval judge. Use the exported submission package with OpenAI's official GDPval judge\n to obtain quality scores.\n- Full document/spreadsheet/slide quality depends on the GDPval runtime image. Thin Python images are useful only for\n plumbing smoke tests.\n\n## Scoring and Submission\n\n- EvalScope writes a local submission folder under the reports directory.\n- The submission contains `deliverable_text` and `deliverable_files` fields in the GDPval dataset format.\n- Official GDPval grading is external. Run OpenAI's official GDPval judge on the exported submission package.\n", "prompt_template": "{question}", "system_prompt": "", "few_shot_prompt_template": "", "aggregation": "mean", "extra_params": { - "max_steps": { - "type": "int", - "description": "Maximum number of agent steps per sample.", - "value": 250 - }, - "command_timeout": { - "type": "float", - "description": "Default per-command timeout in seconds.", - "value": 180.0 - }, - "docker_image": { - "type": "str", - "description": "Docker image used as the per-sample sandbox.", - "value": "evalscope/gdpval:latest" - }, "auto_build_docker_image": { "type": "bool", "description": "Automatically build the default GDPval Docker image if it is missing locally.", "value": true }, - "network_enabled": { - "type": "bool", - "description": "Allow the sandbox to access the network.", - "value": true - }, "download_reference_files": { "type": "bool", "description": "Download each selected sample reference file from the dataset hub before inference.", @@ -55,6 +35,10 @@ } }, "sandbox_config": {}, + "agent_config": { + "strategy": "function_calling", + "max_steps": 250 + }, "category": "agent" }, "statistics": { @@ -162,11 +146,11 @@ "truncated": false }, "readme": { - "en": "# GDPval\n\n\n## Overview\n\nGDPval evaluates whether models can complete realistic economically valuable work tasks and produce requested\ndeliverable files. This adapter targets OpenAI's public 220-task gold subset mirrored on ModelScope as\n`openai-mirror/gdpval`.\n\n## Task Description\n\n- **Task Type**: Agentic professional work / deliverable generation\n- **Input**: A workplace-style task prompt, optionally with reference files\n- **Output**: Final response text and requested files under `deliverable_files/`\n- **Dataset**: OpenAI public GDPval gold subset with 220 tasks\n\n## Key Features\n\n- Uses the native EvalScope `AgentLoopAdapter` with bash and Python execution tools.\n- Loads records and reference files from ModelScope by default.\n- Mounts selected reference files read-only into the sandbox under `/reference_files`.\n- Extracts files written to `deliverable_files/` before sandbox teardown.\n- Generates a GDPval submission package with `deliverable_text` and `deliverable_files` columns.\n\n## Evaluation Notes\n\n- The default Docker image is `evalscope/gdpval:latest` and is built automatically from the bundled Dockerfile when\n missing. Set `extra_params.auto_build_docker_image=false` to require a pre-built image, or override\n `extra_params.docker_image`.\n- `submission_ready` is a local readiness metric: it is 1 when the model produced final text or at least one\n deliverable file. It is not an official GDPval quality score.\n- EvalScope does not run a local GDPval judge. Use the exported submission package with OpenAI's official GDPval judge\n to obtain quality scores.\n- Full document/spreadsheet/slide quality depends on the GDPval runtime image. Thin Python images are useful only for\n plumbing smoke tests.\n\n## Scoring and Submission\n\n- EvalScope writes a local submission folder under the reports directory.\n- The submission contains `deliverable_text` and `deliverable_files` fields in the GDPval dataset format.\n- Official GDPval grading is external. Run OpenAI's official GDPval judge on the exported submission package.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `gdpval` |\n| **Dataset ID** | [openai-mirror/gdpval](https://modelscope.cn/datasets/openai-mirror/gdpval/summary) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `Knowledge`, `MultiTurn` |\n| **Metrics** | `submission_ready` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 220 |\n| Prompt Length (Mean) | 2742.59 chars |\n| Prompt Length (Min/Max) | 1058 / 7160 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"3315415d\",\n \"content\": \"You are an auditor and as part of an audit engagement, you are tasked with reviewing and testing the accuracy of reported Anti-Financial Crime Risk Metrics.\\n\\nThe attached spreadsheet titled ‘Population’ contains Anti-Financial Crime Risk Metr ... [TRUNCATED 2069 chars] ... folder named `deliverable_files` in the sandbox working directory.\\nWe will grade your final message as part of the deliverable, but requested documents, spreadsheets, slides, media,\\nor archives should be actual files in `deliverable_files`.\\n\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n },\n {\n \"name\": \"python_exec\",\n \"description\": \"Execute Python source code inside the sandbox environment. Returns stdout and stderr output.\",\n \"parameters\": {\n \"properties\": {\n \"code\": {\n \"type\": \"string\",\n \"description\": \"Python source code to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"code\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"task_id\": \"83d10b06-26d1-4636-a32c-23f92c57f30b\",\n \"sector\": \"Professional, Scientific, and Technical Services\",\n \"occupation\": \"Accountants and Auditors\",\n \"prompt\": \"You are an auditor and as part of an audit engagement, you are tasked with reviewing and testing the accuracy of reported Anti-Financial Crime Risk Metrics.\\n\\nThe attached spreadsheet titled ‘Population’ contains Anti-Financial Crime Risk Metr ... [TRUNCATED 1526 chars] ... across all Divisions and sub-Divisions.\\n\\n4. Create a new spreadsheet titled ‘Sample’:\\n- Tab 1: Selected sample, copied from the original ‘Population’ sheet, with selected rows marked in column K.\\n- Tab 2: Workings for sample size calculation.\",\n \"reference_files\": [\n \"reference_files/cc781e4dc0985c8eb327a53ec03b5900/Population v2.xlsx\"\n ],\n \"reference_file_urls\": [\n \"https://huggingface.co/datasets/openai/gdpval/resolve/main/reference_files/cc781e4dc0985c8eb327a53ec03b5900/Population%20v2.xlsx\"\n ],\n \"reference_file_hf_uris\": [\n \"hf://datasets/openai/gdpval@main/reference_files/cc781e4dc0985c8eb327a53ec03b5900/Population%20v2.xlsx\"\n ],\n \"reference_paths\": [\n \"reference_files/Population v2.xlsx\"\n ],\n \"sandbox_reference_paths\": [\n \"/reference_files/cc781e4dc0985c8eb327a53ec03b5900/Population v2.xlsx\"\n ],\n \"rubric_pretty\": \"[+2] The submitted deliverable is an Excel workbook file whose basename is 'Sample' (accept .xlsx, .xls, or .xlsm).\\n\\n[+2] The workbook contains a worksheet named exactly 'Sample Size Calculation' (case-insensitive, ignoring surrounding spaces ... [TRUNCATED 4861 chars] ... ntage changes (e.g., |J| ≥ 100%) are made easily identifiable (such as by a separate flag, note, or conditional formatting).\\n\\n[+1] The first worksheet is named 'Sample' (case-insensitive).\\n\\n[+5] Overall formatting and style of the deliverable\",\n \"rubric_json\": \"[{\\\"score\\\": 2, \\\"criterion\\\": \\\"The submitted deliverable is an Excel workbook file whose basename is 'Sample' (accept .xlsx, .xls, or .xlsm).\\\", \\\"required\\\": null, \\\"rubric_item_id\\\": \\\"1d43f1eb-4011-47ac-8ad7-a3c467639a6a\\\", \\\"author_type\\\": \\\"human\\\", \\\" ... [TRUNCATED 11817 chars] ... ull}, {\\\"score\\\": 5, \\\"criterion\\\": \\\"Overall formatting and style of the deliverable\\\", \\\"required\\\": null, \\\"rubric_item_id\\\": \\\"a64588ed-db04-4b8b-b3b8-3674ddcf10d1\\\", \\\"author_type\\\": \\\"human\\\", \\\"tags\\\": [\\\"true\\\"], \\\"read_only\\\": null, \\\"form_content\\\": null}]\",\n \"dataset_id\": \"openai-mirror/gdpval\",\n \"dataset_hub\": \"modelscope\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `max_steps` | `int` | `250` | Maximum number of agent steps per sample. |\n| `command_timeout` | `float` | `180.0` | Default per-command timeout in seconds. |\n| `docker_image` | `str` | `evalscope/gdpval:latest` | Docker image used as the per-sample sandbox. |\n| `auto_build_docker_image` | `bool` | `True` | Automatically build the default GDPval Docker image if it is missing locally. |\n| `network_enabled` | `bool` | `True` | Allow the sandbox to access the network. |\n| `download_reference_files` | `bool` | `True` | Download each selected sample reference file from the dataset hub before inference. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets gdpval \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['gdpval'],\n dataset_args={\n 'gdpval': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# GDPval\n\n\n## 概述\n\nGDPval 用于评估模型是否能够完成具有现实经济价值的工作任务，并生成所要求的交付文件。该适配器针对 OpenAI 公开的 220 个任务的 gold 子集，该子集在 ModelScope 上镜像为 `openai-mirror/gdpval`。\n\n## 任务描述\n\n- **任务类型**：代理式专业工作 / 交付物生成\n- **输入**：职场风格的任务提示，可选附带参考文件\n- **输出**：最终回复文本及位于 `deliverable_files/` 目录下的请求文件\n- **数据集**：OpenAI 公开的 GDPval gold 子集，包含 220 个任务\n\n## 主要特性\n\n- 使用原生 EvalScope 的 `AgentLoopAdapter`，支持 bash 和 Python 执行工具。\n- 默认从 ModelScope 加载记录和参考文件。\n- 将选定的参考文件以只读方式挂载到沙箱中的 `/reference_files` 目录下。\n- 在沙箱销毁前提取写入 `deliverable_files/` 的文件。\n- 生成符合 GDPval 格式的提交包，包含 `deliverable_text` 和 `deliverable_files` 列。\n\n## 评估说明\n\n- 默认 Docker 镜像为 `evalscope/gdpval:latest`，若本地缺失则会根据内置的 Dockerfile 自动构建。设置 `extra_params.auto_build_docker_image=false` 可强制使用预构建镜像，或通过 `extra_params.docker_image` 覆盖默认镜像。\n- `submission_ready` 是一个本地就绪指标：当模型生成了最终文本或至少一个交付文件时，其值为 1。该指标并非官方 GDPval 质量评分。\n- EvalScope 不运行本地 GDPval 评判器。请使用导出的提交包配合 OpenAI 官方 GDPval 评判器获取质量评分。\n- 文档/电子表格/幻灯片等完整文件的质量依赖于 GDPval 运行时镜像。轻量级 Python 镜像仅适用于基础流程的冒烟测试。\n\n## 评分与提交\n\n- EvalScope 会在报告目录下写入一个本地提交文件夹。\n- 提交内容包含符合 GDPval 数据集格式的 `deliverable_text` 和 `deliverable_files` 字段。\n- 官方 GDPval 评分需在外部进行。请对导出的提交包运行 OpenAI 官方 GDPval 评判器。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `gdpval` |\n| **数据集ID** | [openai-mirror/gdpval](https://modelscope.cn/datasets/openai-mirror/gdpval/summary) |\n| **论文** | N/A |\n| **标签** | `Agent`, `Knowledge`, `MultiTurn` |\n| **指标** | `submission_ready` |\n| **默认示例数** | 0-shot |\n| **评估分割** | `train` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 220 |\n| 提示词长度（平均） | 2742.59 字符 |\n| 提示词长度（最小/最大） | 1058 / 7160 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"3315415d\",\n \"content\": \"You are an auditor and as part of an audit engagement, you are tasked with reviewing and testing the accuracy of reported Anti-Financial Crime Risk Metrics.\\n\\nThe attached spreadsheet titled ‘Population’ contains Anti-Financial Crime Risk Metr ... [TRUNCATED 2069 chars] ... folder named `deliverable_files` in the sandbox working directory.\\nWe will grade your final message as part of the deliverable, but requested documents, spreadsheets, slides, media,\\nor archives should be actual files in `deliverable_files`.\\n\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n },\n {\n \"name\": \"python_exec\",\n \"description\": \"Execute Python source code inside the sandbox environment. Returns stdout and stderr output.\",\n \"parameters\": {\n \"properties\": {\n \"code\": {\n \"type\": \"string\",\n \"description\": \"Python source code to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"code\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"task_id\": \"83d10b06-26d1-4636-a32c-23f92c57f30b\",\n \"sector\": \"Professional, Scientific, and Technical Services\",\n \"occupation\": \"Accountants and Auditors\",\n \"prompt\": \"You are an auditor and as part of an audit engagement, you are tasked with reviewing and testing the accuracy of reported Anti-Financial Crime Risk Metrics.\\n\\nThe attached spreadsheet titled ‘Population’ contains Anti-Financial Crime Risk Metr ... [TRUNCATED 1526 chars] ... across all Divisions and sub-Divisions.\\n\\n4. Create a new spreadsheet titled ‘Sample’:\\n- Tab 1: Selected sample, copied from the original ‘Population’ sheet, with selected rows marked in column K.\\n- Tab 2: Workings for sample size calculation.\",\n \"reference_files\": [\n \"reference_files/cc781e4dc0985c8eb327a53ec03b5900/Population v2.xlsx\"\n ],\n \"reference_file_urls\": [\n \"https://huggingface.co/datasets/openai/gdpval/resolve/main/reference_files/cc781e4dc0985c8eb327a53ec03b5900/Population%20v2.xlsx\"\n ],\n \"reference_file_hf_uris\": [\n \"hf://datasets/openai/gdpval@main/reference_files/cc781e4dc0985c8eb327a53ec03b5900/Population%20v2.xlsx\"\n ],\n \"reference_paths\": [\n \"reference_files/Population v2.xlsx\"\n ],\n \"sandbox_reference_paths\": [\n \"/reference_files/cc781e4dc0985c8eb327a53ec03b5900/Population v2.xlsx\"\n ],\n \"rubric_pretty\": \"[+2] The submitted deliverable is an Excel workbook file whose basename is 'Sample' (accept .xlsx, .xls, or .xlsm).\\n\\n[+2] The workbook contains a worksheet named exactly 'Sample Size Calculation' (case-insensitive, ignoring surrounding spaces ... [TRUNCATED 4861 chars] ... ntage changes (e.g., |J| ≥ 100%) are made easily identifiable (such as by a separate flag, note, or conditional formatting).\\n\\n[+1] The first worksheet is named 'Sample' (case-insensitive).\\n\\n[+5] Overall formatting and style of the deliverable\",\n \"rubric_json\": \"[{\\\"score\\\": 2, \\\"criterion\\\": \\\"The submitted deliverable is an Excel workbook file whose basename is 'Sample' (accept .xlsx, .xls, or .xlsm).\\\", \\\"required\\\": null, \\\"rubric_item_id\\\": \\\"1d43f1eb-4011-47ac-8ad7-a3c467639a6a\\\", \\\"author_type\\\": \\\"human\\\", \\\" ... [TRUNCATED 11817 chars] ... ull}, {\\\"score\\\": 5, \\\"criterion\\\": \\\"Overall formatting and style of the deliverable\\\", \\\"required\\\": null, \\\"rubric_item_id\\\": \\\"a64588ed-db04-4b8b-b3b8-3674ddcf10d1\\\", \\\"author_type\\\": \\\"human\\\", \\\"tags\\\": [\\\"true\\\"], \\\"read_only\\\": null, \\\"form_content\\\": null}]\",\n \"dataset_id\": \"openai-mirror/gdpval\",\n \"dataset_hub\": \"modelscope\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板:**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `max_steps` | `int` | `250` | 每个样本的最大代理步骤数。 |\n| `command_timeout` | `float` | `180.0` | 每条命令的默认超时时间（秒）。 |\n| `docker_image` | `str` | `evalscope/gdpval:latest` | 用作每个样本沙箱的 Docker 镜像。 |\n| `auto_build_docker_image` | `bool` | `True` | 若本地缺失默认 GDPval Docker 镜像，则自动构建。 |\n| `network_enabled` | `bool` | `True` | 允许沙箱访问网络。 |\n| `download_reference_files` | `bool` | `True` | 在推理前从数据集中心下载每个选定样本的参考文件。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets gdpval \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['gdpval'],\n dataset_args={\n 'gdpval': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "44951e777aaf601d1b5a0304c2a3aaf3", + "en": "# GDPval\n\n\n## Overview\n\nGDPval evaluates whether models can complete realistic economically valuable work tasks and produce requested\ndeliverable files. This adapter targets OpenAI's public 220-task gold subset mirrored on ModelScope as\n`openai-mirror/gdpval`.\n\n## Task Description\n\n- **Task Type**: Agentic professional work / deliverable generation\n- **Input**: A workplace-style task prompt, optionally with reference files\n- **Output**: Final response text and requested files under `deliverable_files/`\n- **Dataset**: OpenAI public GDPval gold subset with 220 tasks\n\n## Key Features\n\n- Uses the native EvalScope `AgentLoopAdapter` with bash and Python execution tools.\n- Loads records and reference files from ModelScope by default.\n- Mounts selected reference files read-only into the sandbox under `/reference_files`.\n- Extracts files written to `deliverable_files/` before sandbox teardown.\n- Generates a GDPval submission package with `deliverable_text` and `deliverable_files` columns.\n\n## Evaluation Notes\n\n- The default Docker image is built automatically from the bundled Dockerfile into a content-hashed local tag. Set\n `extra_params.auto_build_docker_image=false` to require a pre-built `evalscope/gdpval:latest`. Override the image,\n network, CPU or memory settings through `TaskConfig.sandbox.default_config`.\n- `submission_ready` is a local readiness metric: it is 1 when the model produced final text or at least one\n deliverable file. It is not an official GDPval quality score.\n- EvalScope does not run a local GDPval judge. Use the exported submission package with OpenAI's official GDPval judge\n to obtain quality scores.\n- Full document/spreadsheet/slide quality depends on the GDPval runtime image. Thin Python images are useful only for\n plumbing smoke tests.\n\n## Scoring and Submission\n\n- EvalScope writes a local submission folder under the reports directory.\n- The submission contains `deliverable_text` and `deliverable_files` fields in the GDPval dataset format.\n- Official GDPval grading is external. Run OpenAI's official GDPval judge on the exported submission package.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `gdpval` |\n| **Dataset ID** | [openai-mirror/gdpval](https://modelscope.cn/datasets/openai-mirror/gdpval/summary) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `Knowledge`, `MultiTurn` |\n| **Metrics** | `submission_ready` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 220 |\n| Prompt Length (Mean) | 2742.59 chars |\n| Prompt Length (Min/Max) | 1058 / 7160 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"3315415d\",\n \"content\": \"You are an auditor and as part of an audit engagement, you are tasked with reviewing and testing the accuracy of reported Anti-Financial Crime Risk Metrics.\\n\\nThe attached spreadsheet titled ‘Population’ contains Anti-Financial Crime Risk Metr ... [TRUNCATED 2069 chars] ... folder named `deliverable_files` in the sandbox working directory.\\nWe will grade your final message as part of the deliverable, but requested documents, spreadsheets, slides, media,\\nor archives should be actual files in `deliverable_files`.\\n\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n },\n {\n \"name\": \"python_exec\",\n \"description\": \"Execute Python source code inside the sandbox environment. Returns stdout and stderr output.\",\n \"parameters\": {\n \"properties\": {\n \"code\": {\n \"type\": \"string\",\n \"description\": \"Python source code to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"code\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"task_id\": \"83d10b06-26d1-4636-a32c-23f92c57f30b\",\n \"sector\": \"Professional, Scientific, and Technical Services\",\n \"occupation\": \"Accountants and Auditors\",\n \"prompt\": \"You are an auditor and as part of an audit engagement, you are tasked with reviewing and testing the accuracy of reported Anti-Financial Crime Risk Metrics.\\n\\nThe attached spreadsheet titled ‘Population’ contains Anti-Financial Crime Risk Metr ... [TRUNCATED 1526 chars] ... across all Divisions and sub-Divisions.\\n\\n4. Create a new spreadsheet titled ‘Sample’:\\n- Tab 1: Selected sample, copied from the original ‘Population’ sheet, with selected rows marked in column K.\\n- Tab 2: Workings for sample size calculation.\",\n \"reference_files\": [\n \"reference_files/cc781e4dc0985c8eb327a53ec03b5900/Population v2.xlsx\"\n ],\n \"reference_file_urls\": [\n \"https://huggingface.co/datasets/openai/gdpval/resolve/main/reference_files/cc781e4dc0985c8eb327a53ec03b5900/Population%20v2.xlsx\"\n ],\n \"reference_file_hf_uris\": [\n \"hf://datasets/openai/gdpval@main/reference_files/cc781e4dc0985c8eb327a53ec03b5900/Population%20v2.xlsx\"\n ],\n \"reference_paths\": [\n \"reference_files/Population v2.xlsx\"\n ],\n \"sandbox_reference_paths\": [\n \"/reference_files/cc781e4dc0985c8eb327a53ec03b5900/Population v2.xlsx\"\n ],\n \"rubric_pretty\": \"[+2] The submitted deliverable is an Excel workbook file whose basename is 'Sample' (accept .xlsx, .xls, or .xlsm).\\n\\n[+2] The workbook contains a worksheet named exactly 'Sample Size Calculation' (case-insensitive, ignoring surrounding spaces ... [TRUNCATED 4861 chars] ... ntage changes (e.g., |J| ≥ 100%) are made easily identifiable (such as by a separate flag, note, or conditional formatting).\\n\\n[+1] The first worksheet is named 'Sample' (case-insensitive).\\n\\n[+5] Overall formatting and style of the deliverable\",\n \"rubric_json\": \"[{\\\"score\\\": 2, \\\"criterion\\\": \\\"The submitted deliverable is an Excel workbook file whose basename is 'Sample' (accept .xlsx, .xls, or .xlsm).\\\", \\\"required\\\": null, \\\"rubric_item_id\\\": \\\"1d43f1eb-4011-47ac-8ad7-a3c467639a6a\\\", \\\"author_type\\\": \\\"human\\\", \\\" ... [TRUNCATED 11817 chars] ... ull}, {\\\"score\\\": 5, \\\"criterion\\\": \\\"Overall formatting and style of the deliverable\\\", \\\"required\\\": null, \\\"rubric_item_id\\\": \\\"a64588ed-db04-4b8b-b3b8-3674ddcf10d1\\\", \\\"author_type\\\": \\\"human\\\", \\\"tags\\\": [\\\"true\\\"], \\\"read_only\\\": null, \\\"form_content\\\": null}]\",\n \"dataset_id\": \"openai-mirror/gdpval\",\n \"dataset_hub\": \"modelscope\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `auto_build_docker_image` | `bool` | `True` | Automatically build the default GDPval Docker image if it is missing locally. |\n| `download_reference_files` | `bool` | `True` | Download each selected sample reference file from the dataset hub before inference. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets gdpval \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":250}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['gdpval'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=250,\n ),\n dataset_args={\n 'gdpval': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# GDPval\n\n\n## 概述\n\nGDPval 评估模型是否能够完成具有现实经济价值的工作任务，并生成所要求的交付文件。该适配器针对 OpenAI 公开的 220 个任务的 gold 子集，该子集在 ModelScope 上镜像为 `openai-mirror/gdpval`。\n\n## 任务描述\n\n- **任务类型**：代理式专业工作 / 交付物生成\n- **输入**：职场风格的任务提示，可选附带参考文件\n- **输出**：最终响应文本及位于 `deliverable_files/` 目录下的请求文件\n- **数据集**：OpenAI 公开的 GDPval gold 子集，包含 220 个任务\n\n## 主要特性\n\n- 使用原生 EvalScope 的 `AgentLoopAdapter`，支持 bash 和 Python 执行工具。\n- 默认从 ModelScope 加载记录和参考文件。\n- 将选定的参考文件以只读方式挂载到沙箱中的 `/reference_files` 目录下。\n- 在沙箱销毁前提取写入 `deliverable_files/` 的文件。\n- 生成符合 GDPval 格式的提交包，包含 `deliverable_text` 和 `deliverable_files` 两列。\n\n## 评估说明\n\n- 默认 Docker 镜像会根据捆绑的 Dockerfile 自动构建，并打上基于内容哈希的本地标签。设置 \n `extra_params.auto_build_docker_image=false` 可强制使用预构建的 `evalscope/gdpval:latest` 镜像。可通过 `TaskConfig.sandbox.default_config` 覆盖镜像、网络、CPU 或内存配置。\n- `submission_ready` 是一个本地就绪指标：当模型生成了最终文本或至少一个交付文件时，其值为 1。该指标并非官方 GDPval 质量评分。\n- EvalScope 不运行本地 GDPval 评判器。请使用导出的提交包配合 OpenAI 官方 GDPval 评判器获取质量评分。\n- 完整文档/电子表格/幻灯片的质量依赖于 GDPval 运行时镜像。轻量级 Python 镜像仅适用于基础流程的冒烟测试。\n\n## 评分与提交\n\n- EvalScope 会在报告目录下写入一个本地提交文件夹。\n- 提交内容包含符合 GDPval 数据集格式的 `deliverable_text` 和 `deliverable_files` 字段。\n- 官方 GDPval 评分需在外部进行。请在导出的提交包上运行 OpenAI 官方 GDPval 评判器。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `gdpval` |\n| **数据集ID** | [openai-mirror/gdpval](https://modelscope.cn/datasets/openai-mirror/gdpval/summary) |\n| **论文** | N/A |\n| **标签** | `Agent`, `Knowledge`, `MultiTurn` |\n| **指标** | `submission_ready` |\n| **默认示例数** | 0-shot |\n| **评估分割** | `train` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 220 |\n| 提示词长度（平均） | 2742.59 字符 |\n| 提示词长度（最小/最大） | 1058 / 7160 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"3315415d\",\n \"content\": \"You are an auditor and as part of an audit engagement, you are tasked with reviewing and testing the accuracy of reported Anti-Financial Crime Risk Metrics.\\n\\nThe attached spreadsheet titled ‘Population’ contains Anti-Financial Crime Risk Metr ... [TRUNCATED 2069 chars] ... folder named `deliverable_files` in the sandbox working directory.\\nWe will grade your final message as part of the deliverable, but requested documents, spreadsheets, slides, media,\\nor archives should be actual files in `deliverable_files`.\\n\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n },\n {\n \"name\": \"python_exec\",\n \"description\": \"Execute Python source code inside the sandbox environment. Returns stdout and stderr output.\",\n \"parameters\": {\n \"properties\": {\n \"code\": {\n \"type\": \"string\",\n \"description\": \"Python source code to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"code\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"task_id\": \"83d10b06-26d1-4636-a32c-23f92c57f30b\",\n \"sector\": \"Professional, Scientific, and Technical Services\",\n \"occupation\": \"Accountants and Auditors\",\n \"prompt\": \"You are an auditor and as part of an audit engagement, you are tasked with reviewing and testing the accuracy of reported Anti-Financial Crime Risk Metrics.\\n\\nThe attached spreadsheet titled ‘Population’ contains Anti-Financial Crime Risk Metr ... [TRUNCATED 1526 chars] ... across all Divisions and sub-Divisions.\\n\\n4. Create a new spreadsheet titled ‘Sample’:\\n- Tab 1: Selected sample, copied from the original ‘Population’ sheet, with selected rows marked in column K.\\n- Tab 2: Workings for sample size calculation.\",\n \"reference_files\": [\n \"reference_files/cc781e4dc0985c8eb327a53ec03b5900/Population v2.xlsx\"\n ],\n \"reference_file_urls\": [\n \"https://huggingface.co/datasets/openai/gdpval/resolve/main/reference_files/cc781e4dc0985c8eb327a53ec03b5900/Population%20v2.xlsx\"\n ],\n \"reference_file_hf_uris\": [\n \"hf://datasets/openai/gdpval@main/reference_files/cc781e4dc0985c8eb327a53ec03b5900/Population%20v2.xlsx\"\n ],\n \"reference_paths\": [\n \"reference_files/Population v2.xlsx\"\n ],\n \"sandbox_reference_paths\": [\n \"/reference_files/cc781e4dc0985c8eb327a53ec03b5900/Population v2.xlsx\"\n ],\n \"rubric_pretty\": \"[+2] The submitted deliverable is an Excel workbook file whose basename is 'Sample' (accept .xlsx, .xls, or .xlsm).\\n\\n[+2] The workbook contains a worksheet named exactly 'Sample Size Calculation' (case-insensitive, ignoring surrounding spaces ... [TRUNCATED 4861 chars] ... ntage changes (e.g., |J| ≥ 100%) are made easily identifiable (such as by a separate flag, note, or conditional formatting).\\n\\n[+1] The first worksheet is named 'Sample' (case-insensitive).\\n\\n[+5] Overall formatting and style of the deliverable\",\n \"rubric_json\": \"[{\\\"score\\\": 2, \\\"criterion\\\": \\\"The submitted deliverable is an Excel workbook file whose basename is 'Sample' (accept .xlsx, .xls, or .xlsm).\\\", \\\"required\\\": null, \\\"rubric_item_id\\\": \\\"1d43f1eb-4011-47ac-8ad7-a3c467639a6a\\\", \\\"author_type\\\": \\\"human\\\", \\\" ... [TRUNCATED 11817 chars] ... ull}, {\\\"score\\\": 5, \\\"criterion\\\": \\\"Overall formatting and style of the deliverable\\\", \\\"required\\\": null, \\\"rubric_item_id\\\": \\\"a64588ed-db04-4b8b-b3b8-3674ddcf10d1\\\", \\\"author_type\\\": \\\"human\\\", \\\"tags\\\": [\\\"true\\\"], \\\"read_only\\\": null, \\\"form_content\\\": null}]\",\n \"dataset_id\": \"openai-mirror/gdpval\",\n \"dataset_hub\": \"modelscope\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板:**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `auto_build_docker_image` | `bool` | `True` | 如果本地缺少默认 GDPval Docker 镜像，则自动构建。 |\n| `download_reference_files` | `bool` | `True` | 在推理前从数据集中心下载每个选定样本的参考文件。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets gdpval \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":250}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['gdpval'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=250,\n ),\n dataset_args={\n 'gdpval': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "ae1c469ab3b0e697411df92253ddba25", "needs_translation": false }, - "updated_at": "2026-06-30T18:01:44.292643", - "translation_updated_at": "2026-06-30T18:01:47" -} \ No newline at end of file + "updated_at": "2026-07-10T20:24:32.709243", + "translation_updated_at": "2026-07-10T20:24:42" +} diff --git a/evalscope/evalscope/benchmarks/_meta/gedit.json b/evalscope/evalscope/benchmarks/_meta/gedit.json index dbf77f5..6a94ecc 100644 --- a/evalscope/evalscope/benchmarks/_meta/gedit.json +++ b/evalscope/evalscope/benchmarks/_meta/gedit.json @@ -578,4 +578,4 @@ }, "updated_at": "2026-01-28T17:31:32.267117", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/genai_bench.json b/evalscope/evalscope/benchmarks/_meta/genai_bench.json index bebac2e..a2c1742 100644 --- a/evalscope/evalscope/benchmarks/_meta/genai_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/genai_bench.json @@ -83,4 +83,4 @@ }, "updated_at": "2026-01-28T17:31:32.514638", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/general_arena.json b/evalscope/evalscope/benchmarks/_meta/general_arena.json index 6418ae8..c491e99 100644 --- a/evalscope/evalscope/benchmarks/_meta/general_arena.json +++ b/evalscope/evalscope/benchmarks/_meta/general_arena.json @@ -66,4 +66,4 @@ }, "updated_at": "2026-01-28T17:31:32.233958", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/general_fc.json b/evalscope/evalscope/benchmarks/_meta/general_fc.json index e370616..56d324d 100644 --- a/evalscope/evalscope/benchmarks/_meta/general_fc.json +++ b/evalscope/evalscope/benchmarks/_meta/general_fc.json @@ -204,4 +204,4 @@ }, "updated_at": "2026-05-26T22:46:10.595586", "translation_updated_at": "2026-06-01T14:08:20" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/general_mcq.json b/evalscope/evalscope/benchmarks/_meta/general_mcq.json index b3d4ed5..b72de33 100644 --- a/evalscope/evalscope/benchmarks/_meta/general_mcq.json +++ b/evalscope/evalscope/benchmarks/_meta/general_mcq.json @@ -57,4 +57,4 @@ }, "updated_at": "2026-05-13T12:28:37.931355", "translation_updated_at": "2026-05-13T12:28:43" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/general_qa.json b/evalscope/evalscope/benchmarks/_meta/general_qa.json index 8a3eda1..85c7f1d 100644 --- a/evalscope/evalscope/benchmarks/_meta/general_qa.json +++ b/evalscope/evalscope/benchmarks/_meta/general_qa.json @@ -47,4 +47,4 @@ }, "updated_at": "2026-01-28T17:31:32.237815", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/general_t2i.json b/evalscope/evalscope/benchmarks/_meta/general_t2i.json index 7c91b2c..649a540 100644 --- a/evalscope/evalscope/benchmarks/_meta/general_t2i.json +++ b/evalscope/evalscope/benchmarks/_meta/general_t2i.json @@ -46,4 +46,4 @@ }, "updated_at": "2026-07-02T19:24:57.898727", "translation_updated_at": "2026-07-02T19:26:43" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/general_vmcq.json b/evalscope/evalscope/benchmarks/_meta/general_vmcq.json index e73553d..1ef25fc 100644 --- a/evalscope/evalscope/benchmarks/_meta/general_vmcq.json +++ b/evalscope/evalscope/benchmarks/_meta/general_vmcq.json @@ -47,4 +47,4 @@ }, "updated_at": "2026-05-18T10:23:18.720576", "translation_updated_at": "2026-05-18T10:23:22" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/general_vqa.json b/evalscope/evalscope/benchmarks/_meta/general_vqa.json index b77c15a..d167de3 100644 --- a/evalscope/evalscope/benchmarks/_meta/general_vqa.json +++ b/evalscope/evalscope/benchmarks/_meta/general_vqa.json @@ -48,4 +48,4 @@ }, "updated_at": "2026-05-18T10:23:18.721476", "translation_updated_at": "2026-05-18T10:23:22" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/genia_ner.json b/evalscope/evalscope/benchmarks/_meta/genia_ner.json index 5016678..b401c1b 100644 --- a/evalscope/evalscope/benchmarks/_meta/genia_ner.json +++ b/evalscope/evalscope/benchmarks/_meta/genia_ner.json @@ -151,4 +151,4 @@ }, "updated_at": "2026-01-28T17:31:32.356634", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/gpqa_diamond.json b/evalscope/evalscope/benchmarks/_meta/gpqa_diamond.json index 5b3e616..498b750 100644 --- a/evalscope/evalscope/benchmarks/_meta/gpqa_diamond.json +++ b/evalscope/evalscope/benchmarks/_meta/gpqa_diamond.json @@ -85,4 +85,4 @@ }, "updated_at": "2026-01-28T17:31:32.240215", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/gsm8k.json b/evalscope/evalscope/benchmarks/_meta/gsm8k.json index 5952712..6c76a14 100644 --- a/evalscope/evalscope/benchmarks/_meta/gsm8k.json +++ b/evalscope/evalscope/benchmarks/_meta/gsm8k.json @@ -77,4 +77,4 @@ }, "updated_at": "2026-01-28T17:31:32.243854", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/gsm8k_v.json b/evalscope/evalscope/benchmarks/_meta/gsm8k_v.json index 53ecbb6..443f009 100644 --- a/evalscope/evalscope/benchmarks/_meta/gsm8k_v.json +++ b/evalscope/evalscope/benchmarks/_meta/gsm8k_v.json @@ -146,4 +146,4 @@ }, "updated_at": "2026-01-28T17:31:32.243835", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/hallusion_bench.json b/evalscope/evalscope/benchmarks/_meta/hallusion_bench.json index 5131ee8..4c5e24e 100644 --- a/evalscope/evalscope/benchmarks/_meta/hallusion_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/hallusion_bench.json @@ -154,4 +154,4 @@ }, "updated_at": "2026-01-28T17:31:32.248015", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/halueval.json b/evalscope/evalscope/benchmarks/_meta/halueval.json index d0aa173..e076cf6 100644 --- a/evalscope/evalscope/benchmarks/_meta/halueval.json +++ b/evalscope/evalscope/benchmarks/_meta/halueval.json @@ -102,4 +102,4 @@ }, "updated_at": "2026-01-28T17:31:32.251342", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/harvey_ner.json b/evalscope/evalscope/benchmarks/_meta/harvey_ner.json index 0a8fb20..dae6f97 100644 --- a/evalscope/evalscope/benchmarks/_meta/harvey_ner.json +++ b/evalscope/evalscope/benchmarks/_meta/harvey_ner.json @@ -121,4 +121,4 @@ }, "updated_at": "2026-01-28T17:31:32.357175", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/health_bench.json b/evalscope/evalscope/benchmarks/_meta/health_bench.json index d26e23e..d164068 100644 --- a/evalscope/evalscope/benchmarks/_meta/health_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/health_bench.json @@ -183,4 +183,4 @@ }, "updated_at": "2026-01-28T17:31:32.251794", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/hellaswag.json b/evalscope/evalscope/benchmarks/_meta/hellaswag.json index 58ad96a..4c5b2b2 100644 --- a/evalscope/evalscope/benchmarks/_meta/hellaswag.json +++ b/evalscope/evalscope/benchmarks/_meta/hellaswag.json @@ -80,4 +80,4 @@ }, "updated_at": "2026-01-28T17:31:32.251375", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/hle.json b/evalscope/evalscope/benchmarks/_meta/hle.json index 95228e8..4eaf55a 100644 --- a/evalscope/evalscope/benchmarks/_meta/hle.json +++ b/evalscope/evalscope/benchmarks/_meta/hle.json @@ -468,4 +468,4 @@ }, "updated_at": "2026-01-28T17:31:32.251769", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/hmmt25.json b/evalscope/evalscope/benchmarks/_meta/hmmt25.json index f196542..dbee4fe 100644 --- a/evalscope/evalscope/benchmarks/_meta/hmmt25.json +++ b/evalscope/evalscope/benchmarks/_meta/hmmt25.json @@ -80,4 +80,4 @@ }, "updated_at": "2026-01-28T17:31:32.251564", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/hmmt26.json b/evalscope/evalscope/benchmarks/_meta/hmmt26.json index 76cb3cc..92b9f26 100644 --- a/evalscope/evalscope/benchmarks/_meta/hmmt26.json +++ b/evalscope/evalscope/benchmarks/_meta/hmmt26.json @@ -80,4 +80,4 @@ }, "updated_at": "2026-07-03T16:45:55.609077", "translation_updated_at": "2026-07-03T16:46:16" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/hpdv2.json b/evalscope/evalscope/benchmarks/_meta/hpdv2.json index c3dd561..24fcf2e 100644 --- a/evalscope/evalscope/benchmarks/_meta/hpdv2.json +++ b/evalscope/evalscope/benchmarks/_meta/hpdv2.json @@ -77,4 +77,4 @@ }, "updated_at": "2026-01-28T17:31:32.552031", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/humaneval.json b/evalscope/evalscope/benchmarks/_meta/humaneval.json index 6a98181..9c35d78 100644 --- a/evalscope/evalscope/benchmarks/_meta/humaneval.json +++ b/evalscope/evalscope/benchmarks/_meta/humaneval.json @@ -81,4 +81,4 @@ }, "updated_at": "2026-05-15T16:15:11.844498", "translation_updated_at": "2026-05-15T16:15:18" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/humaneval_plus.json b/evalscope/evalscope/benchmarks/_meta/humaneval_plus.json index 79e4819..a412ad6 100644 --- a/evalscope/evalscope/benchmarks/_meta/humaneval_plus.json +++ b/evalscope/evalscope/benchmarks/_meta/humaneval_plus.json @@ -81,4 +81,4 @@ }, "updated_at": "2026-05-15T16:15:14.210795", "translation_updated_at": "2026-05-15T16:15:18" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/ifbench.json b/evalscope/evalscope/benchmarks/_meta/ifbench.json index 0a04866..24d078b 100644 --- a/evalscope/evalscope/benchmarks/_meta/ifbench.json +++ b/evalscope/evalscope/benchmarks/_meta/ifbench.json @@ -125,4 +125,4 @@ }, "updated_at": "2026-04-02T14:46:06.012381", "translation_updated_at": "2026-04-02T14:46:08Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/ifeval.json b/evalscope/evalscope/benchmarks/_meta/ifeval.json index ad8faa5..f2a1c0f 100644 --- a/evalscope/evalscope/benchmarks/_meta/ifeval.json +++ b/evalscope/evalscope/benchmarks/_meta/ifeval.json @@ -161,4 +161,4 @@ }, "updated_at": "2026-01-28T17:31:32.261879", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/imo_answerbench.json b/evalscope/evalscope/benchmarks/_meta/imo_answerbench.json index 18f9d66..05e1251 100644 --- a/evalscope/evalscope/benchmarks/_meta/imo_answerbench.json +++ b/evalscope/evalscope/benchmarks/_meta/imo_answerbench.json @@ -110,4 +110,4 @@ }, "updated_at": "2026-07-03T15:50:53.448046", "translation_updated_at": "2026-07-03T15:51:01" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/infovqa.json b/evalscope/evalscope/benchmarks/_meta/infovqa.json index 646277e..0f32bd3 100644 --- a/evalscope/evalscope/benchmarks/_meta/infovqa.json +++ b/evalscope/evalscope/benchmarks/_meta/infovqa.json @@ -150,4 +150,4 @@ }, "updated_at": "2026-01-28T17:31:32.263399", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/iquiz.json b/evalscope/evalscope/benchmarks/_meta/iquiz.json index 6d2c5c6..5338127 100644 --- a/evalscope/evalscope/benchmarks/_meta/iquiz.json +++ b/evalscope/evalscope/benchmarks/_meta/iquiz.json @@ -90,4 +90,4 @@ }, "updated_at": "2026-01-28T17:31:32.263903", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/jnlpba.json b/evalscope/evalscope/benchmarks/_meta/jnlpba.json index 33f652c..1a05b5e 100644 --- a/evalscope/evalscope/benchmarks/_meta/jnlpba.json +++ b/evalscope/evalscope/benchmarks/_meta/jnlpba.json @@ -105,4 +105,4 @@ }, "updated_at": "2026-01-28T17:31:32.360641", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/jnlpba_rare.json b/evalscope/evalscope/benchmarks/_meta/jnlpba_rare.json index 8ecbf50..ffa6f36 100644 --- a/evalscope/evalscope/benchmarks/_meta/jnlpba_rare.json +++ b/evalscope/evalscope/benchmarks/_meta/jnlpba_rare.json @@ -113,4 +113,4 @@ }, "updated_at": "2026-01-28T17:31:32.362074", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/k2_verifier.json b/evalscope/evalscope/benchmarks/_meta/k2_verifier.json index 854d8c4..3865983 100644 --- a/evalscope/evalscope/benchmarks/_meta/k2_verifier.json +++ b/evalscope/evalscope/benchmarks/_meta/k2_verifier.json @@ -204,4 +204,4 @@ }, "updated_at": "2026-05-26T22:52:31.205423", "translation_updated_at": "2026-05-26T22:52:34" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/kimi_verifier.json b/evalscope/evalscope/benchmarks/_meta/kimi_verifier.json index 5167987..a736c1d 100644 --- a/evalscope/evalscope/benchmarks/_meta/kimi_verifier.json +++ b/evalscope/evalscope/benchmarks/_meta/kimi_verifier.json @@ -100,4 +100,4 @@ }, "updated_at": "2026-05-26T23:30:17.224036", "translation_updated_at": "2026-05-26T23:30:19" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/kina.json b/evalscope/evalscope/benchmarks/_meta/kina.json new file mode 100644 index 0000000..788faec --- /dev/null +++ b/evalscope/evalscope/benchmarks/_meta/kina.json @@ -0,0 +1,89 @@ +{ + "meta": { + "pretty_name": "KINA", + "dataset_id": "evalscope/KINA", + "paper_url": "https://www.2077ai.com/kina", + "tags": [ + "Knowledge", + "MCQ" + ], + "metrics": [ + "acc" + ], + "few_shot_num": 0, + "eval_split": "test", + "train_split": "", + "subset_list": [ + "default" + ], + "description": "\n## Overview\n\nKINA (Knowledge Index of Noah's Ark) is a high-density multidisciplinary knowledge benchmark for evaluating whether large language models can solve expert-level questions across 261 fine-grained disciplines. It is the first benchmark to incorporate disciplinary representativeness as a core design principle.\n\n## Task Description\n\n- **Task Type**: Multiple-Choice Question Answering (MCQ)\n- **Input**: A discipline-specific question with up to 10 lettered options (A–J)\n- **Output**: A single correct answer letter (A–J)\n- **Domains**: 261 disciplines spanning Agronomy, Medicine, Engineering, Humanities, Natural Sciences, and more\n\n## Key Features\n\n- 899 test questions covering 261 fine-grained disciplines\n- Each question has a unique correct answer among up to 10 options (A–J)\n- Includes per-option explanations for training / analysis (not shown to the model)\n- Designed to test deep domain knowledge, not retrieval or commonsense reasoning\n- Introduced at 2077AI with a focus on disciplinary representativeness\n\n## Evaluation Notes\n\n- Default evaluation uses the **test** split (899 samples)\n- Primary metric: **Accuracy** (acc) — Pass@1 for single-inference mode\n- 0-shot Chain-of-Thought (CoT) evaluation, answer extracted from ``ANSWER: [LETTER]`` marker\n- Discipline metadata is stored per-sample and available in review output; no per-discipline subset grouping\n- [GitHub](https://github.com/weihao1115/KINA-Benchmark)\n", + "prompt_template": "Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of {letters}. Think step by step before answering.\n\n{question}\n\n{choices}", + "system_prompt": "", + "few_shot_prompt_template": "", + "aggregation": "mean", + "extra_params": {}, + "sandbox_config": {}, + "category": "llm" + }, + "statistics": { + "total_samples": 899, + "subset_stats": [ + { + "name": "default", + "sample_count": 899, + "prompt_length_mean": 3280.88, + "prompt_length_min": 482, + "prompt_length_max": 22536, + "prompt_length_std": 2186.45, + "target_length_mean": 1 + } + ], + "prompt_length": { + "mean": 3280.88, + "min": 482, + "max": 22536, + "std": 2186.45 + }, + "target_length_mean": 1, + "computed_at": "2026-07-06T17:23:58.648037" + }, + "sample_example": { + "data": { + "input": [ + { + "id": "4750dae8", + "content": "Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of A,B,C,D,E,F,G,H,I,J. Think step by step before answering.\n\nUnder con ... [TRUNCATED 1998 chars] ... it more economically sustainable under concentrate-restricted conditions.\nJ) Choose barn-dried hay because its intact fiber structure significantly increases milk fat percentage, making it more suitable for producing high-fat dairy products." + } + ], + "choices": [ + "Silage, because anaerobic fermentation preserves soluble carbohydrates, true protein, and vitamins effectively, resulting in higher metabolizable energy density, superior palatability, and greater dry matter intake(DMI), thereby helping sustain milk yield when dietary concentrate is limited.", + "Barn-dried hay, as it promotes higher DMI, enabling adequate nutrient intake and improving nitrogen utilization efficiency despite its lower crude protein concentration.", + "Both forages are functionally equivalent and can be substituted on an equal dry matter basis, as they are both classified as roughages and exert no significant differential effect on lactation performance.", + "Barn-dried hay, owing to its physically effective fiber structure and high lignin content, which enhance rumination activity and mitigate the risk of subacute ruminal acidosis.", + "Silage, due to its high moisture content, which reduces voluntary water consumption and contributes to on-farm water conservation.", + "Choose barn-dried hay because it contains no moisture, has a high dry matter content, and is therefore more \"nutrient-concentrated\" than wet silage.", + "Choose silage because it contains probiotics that can directly improve gut health in dairy cows and serve as a protein source to replace concentrate.", + "Choose silage. In southern China's rainy climate, hay is prone to mold growth and aflatoxin contamination, whereas silage avoids this risk and ensures raw milk safety-particularly important when concentrate supply is limited and reliance on safe forage is critical.", + "Choose silage because it can be produced locally(e.g., whole-plant corn), harvested and stored mechanically, and offers lower cost per unit of nutrient compared to purchased high-quality hay, making it more economically sustainable under concentrate-restricted conditions.", + "Choose barn-dried hay because its intact fiber structure significantly increases milk fat percentage, making it more suitable for producing high-fat dairy products." + ], + "target": "B", + "id": 0, + "group_id": 0, + "metadata": { + "index": 0, + "discipline": "Agronomy/Animal Husbandry/Animal Nutrition and Feed Science" + } + }, + "subset": "default", + "truncated": false + }, + "readme": { + "en": "# KINA\n\n\n## Overview\n\nKINA (Knowledge Index of Noah's Ark) is a high-density multidisciplinary knowledge benchmark for evaluating whether large language models can solve expert-level questions across 261 fine-grained disciplines. It is the first benchmark to incorporate disciplinary representativeness as a core design principle.\n\n## Task Description\n\n- **Task Type**: Multiple-Choice Question Answering (MCQ)\n- **Input**: A discipline-specific question with up to 10 lettered options (A–J)\n- **Output**: A single correct answer letter (A–J)\n- **Domains**: 261 disciplines spanning Agronomy, Medicine, Engineering, Humanities, Natural Sciences, and more\n\n## Key Features\n\n- 899 test questions covering 261 fine-grained disciplines\n- Each question has a unique correct answer among up to 10 options (A–J)\n- Includes per-option explanations for training / analysis (not shown to the model)\n- Designed to test deep domain knowledge, not retrieval or commonsense reasoning\n- Introduced at 2077AI with a focus on disciplinary representativeness\n\n## Evaluation Notes\n\n- Default evaluation uses the **test** split (899 samples)\n- Primary metric: **Accuracy** (acc) — Pass@1 for single-inference mode\n- 0-shot Chain-of-Thought (CoT) evaluation, answer extracted from ``ANSWER: [LETTER]`` marker\n- Discipline metadata is stored per-sample and available in review output; no per-discipline subset grouping\n- [GitHub](https://github.com/weihao1115/KINA-Benchmark)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `kina` |\n| **Dataset ID** | [evalscope/KINA](https://modelscope.cn/datasets/evalscope/KINA/summary) |\n| **Paper** | [Paper](https://www.2077ai.com/kina) |\n| **Tags** | `Knowledge`, `MCQ` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 899 |\n| Prompt Length (Mean) | 3280.88 chars |\n| Prompt Length (Min/Max) | 482 / 22536 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"4750dae8\",\n \"content\": \"Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of A,B,C,D,E,F,G,H,I,J. Think step by step before answering.\\n\\nUnder con ... [TRUNCATED 1998 chars] ... it more economically sustainable under concentrate-restricted conditions.\\nJ) Choose barn-dried hay because its intact fiber structure significantly increases milk fat percentage, making it more suitable for producing high-fat dairy products.\"\n }\n ],\n \"choices\": [\n \"Silage, because anaerobic fermentation preserves soluble carbohydrates, true protein, and vitamins effectively, resulting in higher metabolizable energy density, superior palatability, and greater dry matter intake(DMI), thereby helping sustain milk yield when dietary concentrate is limited.\",\n \"Barn-dried hay, as it promotes higher DMI, enabling adequate nutrient intake and improving nitrogen utilization efficiency despite its lower crude protein concentration.\",\n \"Both forages are functionally equivalent and can be substituted on an equal dry matter basis, as they are both classified as roughages and exert no significant differential effect on lactation performance.\",\n \"Barn-dried hay, owing to its physically effective fiber structure and high lignin content, which enhance rumination activity and mitigate the risk of subacute ruminal acidosis.\",\n \"Silage, due to its high moisture content, which reduces voluntary water consumption and contributes to on-farm water conservation.\",\n \"Choose barn-dried hay because it contains no moisture, has a high dry matter content, and is therefore more \\\"nutrient-concentrated\\\" than wet silage.\",\n \"Choose silage because it contains probiotics that can directly improve gut health in dairy cows and serve as a protein source to replace concentrate.\",\n \"Choose silage. In southern China's rainy climate, hay is prone to mold growth and aflatoxin contamination, whereas silage avoids this risk and ensures raw milk safety-particularly important when concentrate supply is limited and reliance on safe forage is critical.\",\n \"Choose silage because it can be produced locally(e.g., whole-plant corn), harvested and stored mechanically, and offers lower cost per unit of nutrient compared to purchased high-quality hay, making it more economically sustainable under concentrate-restricted conditions.\",\n \"Choose barn-dried hay because its intact fiber structure significantly increases milk fat percentage, making it more suitable for producing high-fat dairy products.\"\n ],\n \"target\": \"B\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"index\": 0,\n \"discipline\": \"Agronomy/Animal Husbandry/Animal Nutrition and Feed Science\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\nAnswer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of {letters}. Think step by step before answering.\n\n{question}\n\n{choices}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets kina \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['kina'],\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# KINA\n\n\n## 概述\n\nKINA（Knowledge Index of Noah's Ark，诺亚方舟知识索引）是一个高密度、多学科的知识基准测试，用于评估大语言模型能否解答横跨261个细粒度学科的专家级问题。它是首个将“学科代表性”作为核心设计原则的基准测试。\n\n## 任务描述\n\n- **任务类型**：多项选择题问答（MCQ）\n- **输入**：一个特定学科的问题，附带最多10个字母选项（A–J）\n- **输出**：单个正确答案字母（A–J）\n- **领域范围**：涵盖农学、医学、工程学、人文学科、自然科学等共261个学科\n\n## 核心特性\n\n- 包含899道测试题，覆盖261个细粒度学科\n- 每道题在最多10个选项（A–J）中仅有一个正确答案\n- 提供每个选项的解释（用于训练/分析，不对模型展示）\n- 旨在测试深层领域知识，而非检索能力或常识推理\n- 在2077AI首次提出，强调学科代表性\n\n## 评估说明\n\n- 默认使用 **test** 划分进行评估（899个样本）\n- 主要指标：**准确率**（acc）—— 单次推理模式下的 Pass@1\n- 采用0-shot思维链（CoT）评估方式，从 ``ANSWER: [LETTER]`` 标记中提取答案\n- 每个样本均附带学科元数据，可在评估结果中查看；但未按学科划分子集\n- [GitHub](https://github.com/weihao1115/KINA-Benchmark)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `kina` |\n| **数据集ID** | [evalscope/KINA](https://modelscope.cn/datasets/evalscope/KINA/summary) |\n| **论文** | [Paper](https://www.2077ai.com/kina) |\n| **标签** | `Knowledge`, `MCQ` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 899 |\n| 提示词长度（平均） | 3280.88 字符 |\n| 提示词长度（最小/最大） | 482 / 22536 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"4750dae8\",\n \"content\": \"Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of A,B,C,D,E,F,G,H,I,J. Think step by step before answering.\\n\\nUnder con ... [TRUNCATED 1998 chars] ... it more economically sustainable under concentrate-restricted conditions.\\nJ) Choose barn-dried hay because its intact fiber structure significantly increases milk fat percentage, making it more suitable for producing high-fat dairy products.\"\n }\n ],\n \"choices\": [\n \"Silage, because anaerobic fermentation preserves soluble carbohydrates, true protein, and vitamins effectively, resulting in higher metabolizable energy density, superior palatability, and greater dry matter intake(DMI), thereby helping sustain milk yield when dietary concentrate is limited.\",\n \"Barn-dried hay, as it promotes higher DMI, enabling adequate nutrient intake and improving nitrogen utilization efficiency despite its lower crude protein concentration.\",\n \"Both forages are functionally equivalent and can be substituted on an equal dry matter basis, as they are both classified as roughages and exert no significant differential effect on lactation performance.\",\n \"Barn-dried hay, owing to its physically effective fiber structure and high lignin content, which enhance rumination activity and mitigate the risk of subacute ruminal acidosis.\",\n \"Silage, due to its high moisture content, which reduces voluntary water consumption and contributes to on-farm water conservation.\",\n \"Choose barn-dried hay because it contains no moisture, has a high dry matter content, and is therefore more \\\"nutrient-concentrated\\\" than wet silage.\",\n \"Choose silage because it contains probiotics that can directly improve gut health in dairy cows and serve as a protein source to replace concentrate.\",\n \"Choose silage. In southern China's rainy climate, hay is prone to mold growth and aflatoxin contamination, whereas silage avoids this risk and ensures raw milk safety-particularly important when concentrate supply is limited and reliance on safe forage is critical.\",\n \"Choose silage because it can be produced locally(e.g., whole-plant corn), harvested and stored mechanically, and offers lower cost per unit of nutrient compared to purchased high-quality hay, making it more economically sustainable under concentrate-restricted conditions.\",\n \"Choose barn-dried hay because its intact fiber structure significantly increases milk fat percentage, making it more suitable for producing high-fat dairy products.\"\n ],\n \"target\": \"B\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"index\": 0,\n \"discipline\": \"Agronomy/Animal Husbandry/Animal Nutrition and Feed Science\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\nAnswer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of {letters}. Think step by step before answering.\n\n{question}\n\n{choices}\n```\n\n## 使用方法\n\n### 使用命令行（CLI）\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets kina \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['kina'],\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "85715233f8335b7a1eee759384493330", + "needs_translation": false + }, + "updated_at": "2026-07-06T18:48:45.539691", + "translation_updated_at": "2026-07-06T18:49:04" +} diff --git a/evalscope/evalscope/benchmarks/_meta/librispeech.json b/evalscope/evalscope/benchmarks/_meta/librispeech.json index 1cf2664..09b7a42 100644 --- a/evalscope/evalscope/benchmarks/_meta/librispeech.json +++ b/evalscope/evalscope/benchmarks/_meta/librispeech.json @@ -118,4 +118,4 @@ }, "updated_at": "2026-01-28T17:31:32.265356", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/live_code_bench.json b/evalscope/evalscope/benchmarks/_meta/live_code_bench.json index 32d1bdc..eda7ac0 100644 --- a/evalscope/evalscope/benchmarks/_meta/live_code_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/live_code_bench.json @@ -122,4 +122,4 @@ }, "updated_at": "2026-05-21T12:31:06.519430", "translation_updated_at": "2026-05-21T12:33:58" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/locomo.json b/evalscope/evalscope/benchmarks/_meta/locomo.json index cee33f9..dd58b21 100644 --- a/evalscope/evalscope/benchmarks/_meta/locomo.json +++ b/evalscope/evalscope/benchmarks/_meta/locomo.json @@ -95,4 +95,4 @@ }, "updated_at": "2026-06-17T16:13:40.132287", "translation_updated_at": "2026-06-17T16:13:44" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/logi_qa.json b/evalscope/evalscope/benchmarks/_meta/logi_qa.json index cf27257..fcc384a 100644 --- a/evalscope/evalscope/benchmarks/_meta/logi_qa.json +++ b/evalscope/evalscope/benchmarks/_meta/logi_qa.json @@ -77,4 +77,4 @@ }, "updated_at": "2026-01-28T17:31:32.267330", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/longbench_v2.json b/evalscope/evalscope/benchmarks/_meta/longbench_v2.json index 6e118cc..eab4f0a 100644 --- a/evalscope/evalscope/benchmarks/_meta/longbench_v2.json +++ b/evalscope/evalscope/benchmarks/_meta/longbench_v2.json @@ -106,4 +106,4 @@ }, "updated_at": "2026-03-18T16:52:16.189942", "translation_updated_at": "2026-03-18T16:53:33Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/longmemeval.json b/evalscope/evalscope/benchmarks/_meta/longmemeval.json index 1e4448a..3edfd3f 100644 --- a/evalscope/evalscope/benchmarks/_meta/longmemeval.json +++ b/evalscope/evalscope/benchmarks/_meta/longmemeval.json @@ -150,4 +150,4 @@ }, "updated_at": "2026-06-16T17:43:24.671815", "translation_updated_at": "2026-06-16T17:43:28" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/maritime_bench.json b/evalscope/evalscope/benchmarks/_meta/maritime_bench.json index fccb0a0..d2dfc67 100644 --- a/evalscope/evalscope/benchmarks/_meta/maritime_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/maritime_bench.json @@ -77,4 +77,4 @@ }, "updated_at": "2026-01-28T17:31:32.272066", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/maritime_ocr_bench.json b/evalscope/evalscope/benchmarks/_meta/maritime_ocr_bench.json index abb89a5..3b4b20b 100644 --- a/evalscope/evalscope/benchmarks/_meta/maritime_ocr_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/maritime_ocr_bench.json @@ -323,4 +323,4 @@ }, "updated_at": "2026-06-04T11:17:56.879310", "translation_updated_at": "2026-06-04T11:17:59" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/math_500.json b/evalscope/evalscope/benchmarks/_meta/math_500.json index 80a9fcf..554c14f 100644 --- a/evalscope/evalscope/benchmarks/_meta/math_500.json +++ b/evalscope/evalscope/benchmarks/_meta/math_500.json @@ -119,4 +119,4 @@ }, "updated_at": "2026-01-28T17:31:32.273890", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/math_qa.json b/evalscope/evalscope/benchmarks/_meta/math_qa.json index 64c185b..bbb6c78 100644 --- a/evalscope/evalscope/benchmarks/_meta/math_qa.json +++ b/evalscope/evalscope/benchmarks/_meta/math_qa.json @@ -81,4 +81,4 @@ }, "updated_at": "2026-01-28T17:31:32.275181", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/math_verse.json b/evalscope/evalscope/benchmarks/_meta/math_verse.json index 536d626..497d9d6 100644 --- a/evalscope/evalscope/benchmarks/_meta/math_verse.json +++ b/evalscope/evalscope/benchmarks/_meta/math_verse.json @@ -330,4 +330,4 @@ }, "updated_at": "2026-01-28T17:31:32.274414", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/math_vision.json b/evalscope/evalscope/benchmarks/_meta/math_vision.json index 5e1c164..2ba0449 100644 --- a/evalscope/evalscope/benchmarks/_meta/math_vision.json +++ b/evalscope/evalscope/benchmarks/_meta/math_vision.json @@ -331,4 +331,4 @@ }, "updated_at": "2026-01-28T17:31:32.276562", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/math_vista.json b/evalscope/evalscope/benchmarks/_meta/math_vista.json index 35d7174..57aa955 100644 --- a/evalscope/evalscope/benchmarks/_meta/math_vista.json +++ b/evalscope/evalscope/benchmarks/_meta/math_vista.json @@ -170,4 +170,4 @@ }, "updated_at": "2026-01-28T17:31:32.288551", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mbpp.json b/evalscope/evalscope/benchmarks/_meta/mbpp.json index 6c4bf75..4f27302 100644 --- a/evalscope/evalscope/benchmarks/_meta/mbpp.json +++ b/evalscope/evalscope/benchmarks/_meta/mbpp.json @@ -84,4 +84,4 @@ }, "updated_at": "2026-05-15T16:15:14.222936", "translation_updated_at": "2026-05-15T16:15:18" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mbpp_plus.json b/evalscope/evalscope/benchmarks/_meta/mbpp_plus.json index 29f57d1..08ae16a 100644 --- a/evalscope/evalscope/benchmarks/_meta/mbpp_plus.json +++ b/evalscope/evalscope/benchmarks/_meta/mbpp_plus.json @@ -86,4 +86,4 @@ }, "updated_at": "2026-05-15T16:15:11.792222", "translation_updated_at": "2026-05-15T16:15:18" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mcp_atlas.json b/evalscope/evalscope/benchmarks/_meta/mcp_atlas.json index f832681..ded2bd1 100644 --- a/evalscope/evalscope/benchmarks/_meta/mcp_atlas.json +++ b/evalscope/evalscope/benchmarks/_meta/mcp_atlas.json @@ -33,11 +33,6 @@ "description": "Skip tasks whose ground-truth trajectory uses MCP servers that are not currently enabled.", "value": true }, - "max_steps": { - "type": "int", - "description": "Maximum number of EvalScope agent loop steps per sample.", - "value": 100 - }, "max_tool_calls": { "type": "int", "description": "Maximum MCP tool calls allowed per sample.", @@ -65,6 +60,10 @@ } }, "sandbox_config": {}, + "agent_config": { + "strategy": "function_calling", + "max_steps": 100 + }, "category": "agent" }, "statistics": { @@ -330,11 +329,11 @@ "truncated": false }, "readme": { - "en": "# MCP-Atlas\n\n## Overview\n\nMCP-Atlas is a Scale AI benchmark for evaluating tool-use competency with real Model Context Protocol\n(MCP) servers. It contains public tasks with prompts, allowed tool lists, ground-truth tool trajectories,\nand expert claims used for LLM-as-judge coverage scoring.\n\n## Task Description\n\n- **Task Type**: Tool-use agent benchmark\n- **Input**: Natural-language task prompt plus a per-task MCP tool allowlist\n- **Output**: Final task answer generated after using MCP tools when useful\n- **Grading**: LLM judge checks whether the final response fulfills each expert-defined claim\n\n## Key Features\n\n- Uses EvalScope's native AgentLoop rather than MCP-Atlas's `mcp_eval` completion service.\n- Connects directly to the MCP-Atlas `agent-environment` HTTP service, which must expose `/enabled-servers`,\n `/list-tools`, and `/call-tool`.\n- Filters tasks by currently enabled MCP servers by default, matching MCP-Atlas's public-script behavior for\n environments without every external API key configured.\n- Exposes only the task's `ENABLED_TOOLS` to the model to avoid advertising hundreds of tools at once.\n- Short-circuits repeated calls to MCP servers that hit transport-level failures inside the same sample.\n- Reports mean `coverage_score` and `pass_rate` with a configurable pass threshold.\n\n## Evaluation Notes\n\n- Start the MCP-Atlas `agent-environment` Docker service before running this benchmark. The default URL is\n `http://localhost:1984`.\n- This EvalScope-native adapter is intended to be maintainable inside EvalScope. It is not claimed to be\n leaderboard-equivalent unless the current Scale leaderboard harness settings are separately verified.\n- Full public-set coverage requires configuring the external API keys and service data required by MCP-Atlas.\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `mcp_atlas` |\n| **Dataset ID** | [ScaleAI/MCP-Atlas](https://modelscope.cn/datasets/ScaleAI/MCP-Atlas/summary) |\n| **Paper** | [Paper](https://static.scale.com/uploads/674f4cc7a74e35bcaae1c29a/MCP_Atlas.pdf) |\n| **Tags** | `Agent`, `MultiTurn` |\n| **Metrics** | `coverage_score`, `pass` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 89 |\n| Prompt Length (Mean) | 291.67 chars |\n| Prompt Length (Min/Max) | 133 / 579 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"5dc7f8ae\",\n \"content\": \"I've been working on a local project for telling stories where I've already built a few react components. I want to check the required dependency version for the component that has the fewest lines of code.\"\n }\n ],\n \"target\": \"[\\\"['The component with the fewest lines of code is AspectRatio', 'The AspectRatio component uses `@radix-ui/react-aspect-ratio`', 'The required dependency version for `@radix-ui/react-aspect-ratio` is\\\\\\\\n \\\\\\\"^1.0.3\\\\\\\"']\\\"]\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"filesystem_read_multiple_files\",\n \"description\": \"Read the contents of multiple files simultaneously. This is more efficient than reading files one by one when you need to analyze or compare multiple files. Each file's content is returned with its path as a reference. Failed reads for individual files won't stop the entire operation. Only works within allowed directories.\",\n \"parameters\": {\n \"properties\": {\n \"paths\": {\n \"type\": \"array\",\n \"description\": \"Array of file paths to read. Each path must be a string pointing to a valid file within allowed directories.\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"paths\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"filesystem_directory_tree\",\n \"description\": \"Get a recursive tree view of files and directories as a JSON structure. Each entry includes 'name', 'type' (file/directory), and 'children' for directories. Files have no children array, while directories always have a children array (which may be empty). The output is formatted with 2-space indentation for readability. Only works within allowed directories.\",\n \"parameters\": {\n \"properties\": {\n \"path\": {\n \"type\": \"string\"\n },\n \"excludePatterns\": {\n \"type\": \"array\",\n \"default\": [],\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"filesystem_list_allowed_directories\",\n \"description\": \"Returns the list of directories that this server is allowed to access. Subdirectories within these allowed directories are also accessible. Use this to understand which directories and their nested paths are available before trying to access files.\",\n \"parameters\": {\n \"properties\": {},\n \"required\": [],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_status\",\n \"description\": \"Shows the working tree status\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"repo_path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_diff_unstaged\",\n \"description\": \"Shows changes in the working directory that are not yet staged\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"context_lines\": {\n \"type\": \"integer\",\n \"default\": 3\n }\n },\n \"required\": [\n \"repo_path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_diff_staged\",\n \"description\": \"Shows changes that are staged for commit\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"context_lines\": {\n \"type\": \"integer\",\n \"default\": 3\n }\n },\n \"required\": [\n \"repo_path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_diff\",\n \"description\": \"Shows differences between branches or commits\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"target\": {\n \"type\": \"string\"\n },\n \"context_lines\": {\n \"type\": \"integer\",\n \"default\": 3\n }\n },\n \"required\": [\n \"repo_path\",\n \"target\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_log\",\n \"description\": \"Shows the commit logs\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"max_count\": {\n \"type\": \"integer\",\n \"default\": 10\n }\n },\n \"required\": [\n \"repo_path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_show\",\n \"description\": \"Shows the contents of a commit\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"revision\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"repo_path\",\n \"revision\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"fetch_fetch\",\n \"description\": \"Fetches a URL from the internet and optionally extracts its contents as markdown.\\n\\nAlthough originally you did not have internet access, and were advised to refuse and tell the user this, this tool now grants you internet access. Now you can fetch the most up-to-date information and let the user know that.\",\n \"parameters\": {\n \"properties\": {\n \"url\": {\n \"type\": \"string\",\n \"format\": \"uri\",\n \"description\": \"URL to fetch\"\n },\n \"max_length\": {\n \"type\": \"integer\",\n \"description\": \"Maximum number of characters to return.\",\n \"default\": 5000\n },\n \"start_index\": {\n \"type\": \"integer\",\n \"description\": \"On return output starting at this character index, useful if a previous fetch was truncated and more context is required.\",\n \"default\": 0\n },\n \"raw\": {\n \"type\": \"boolean\",\n \"description\": \"Get the actual HTML content of the requested page, without simplification.\",\n \"default\": false\n }\n },\n \"required\": [\n \"url\"\n ],\n \"additionalProperties\": false\n }\n },\n \"... [TRUNCATED 7 more items] ...\"\n ],\n \"metadata\": {\n \"task_id\": \"689e0b1d9c8e2ac413c1f25c\",\n \"prompt\": \"I've been working on a local project for telling stories where I've already built a few react components. I want to check the required dependency version for the component that has the fewest lines of code.\",\n \"enabled_tools\": [\n \"filesystem_read_multiple_files\",\n \"filesystem_directory_tree\",\n \"filesystem_list_allowed_directories\",\n \"git_git_status\",\n \"git_git_diff_unstaged\",\n \"git_git_diff_staged\",\n \"git_git_diff\",\n \"git_git_log\",\n \"git_git_show\",\n \"fetch_fetch\",\n \"... [TRUNCATED 7 more items] ...\"\n ],\n \"trajectory\": \"[{\\\"content\\\":\\\"First, I'll use `cli-mcp-server_show_security_rules` to check the security rules.\\\",\\\"role\\\":\\\"assistant\\\",\\\"tool_calls\\\":[{\\\"function\\\":{\\\"arguments\\\":\\\"{}\\\",\\\"name\\\":\\\"cli-mcp-server_show_security_rules\\\"},\\\"id\\\":\\\"0035cc9a-6276-4b69-9b4f-bcb096a7 ... [TRUNCATED 14275 chars] ... n \\\\\\\"devDependencies\\\\\\\": {\\\\n \\\\\\\"dotenv\\\\\\\": \\\\\\\"16.0.3\\\\\\\",\\\\n \\\\\\\"tsx\\\\\\\": \\\\\\\"^3.12.8\\\\\\\"\\\\n }\\\\n}\\\\n\\\",\\\"type\\\":\\\"text\\\"},{\\\"text\\\":\\\"\\\\nCommand completed with return code: 0\\\",\\\"type\\\":\\\"text\\\"}],\\\"role\\\":\\\"tool\\\",\\\"tool_call_id\\\":\\\"6fb02c16-98f1-4a8d-843b-93d3f3f4d02c\\\"}]\",\n \"gtfa_claims\": [\n \"['The component with the fewest lines of code is AspectRatio', 'The AspectRatio component uses `@radix-ui/react-aspect-ratio`', 'The required dependency version for `@radix-ui/react-aspect-ratio` is\\\\n \\\"^1.0.3\\\"']\"\n ],\n \"required_servers\": [\n \"cli-mcp-server\",\n \"filesystem\"\n ],\n \"mcp_server_url\": \"http://localhost:1984\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `mcp_server_url` | `str` | `http://localhost:1984` | MCP-Atlas agent-environment base URL. |\n| `filter_enabled_servers` | `bool` | `True` | Skip tasks whose ground-truth trajectory uses MCP servers that are not currently enabled. |\n| `max_steps` | `int` | `100` | Maximum number of EvalScope agent loop steps per sample. |\n| `max_tool_calls` | `int` | `100` | Maximum MCP tool calls allowed per sample. |\n| `request_timeout` | `float` | `60.0` | Timeout in seconds for MCP tool calls. |\n| `list_tools_timeout` | `float` | `180.0` | Timeout in seconds for MCP server preflight and list-tools requests. |\n| `use_system_prompt` | `bool` | `False` | Prepend the MCP-Atlas optional system prompt to every sample. |\n| `pass_threshold` | `float` | `0.75` | Coverage score threshold used to compute pass rate. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets mcp_atlas \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['mcp_atlas'],\n dataset_args={\n 'mcp_atlas': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# MCP-Atlas\n\n## 概述\n\nMCP-Atlas 是 Scale AI 推出的一项基准测试，用于评估模型在真实 Model Context Protocol (MCP) 服务器环境下的工具使用能力。该基准包含公开任务，每个任务提供提示词、允许使用的工具列表、真实工具调用轨迹（ground-truth tool trajectories），以及用于 LLM-as-judge 覆盖率评分的专家声明（expert claims）。\n\n## 任务描述\n\n- **任务类型**：工具使用智能体基准测试\n- **输入**：自然语言任务提示词，以及每个任务对应的 MCP 工具白名单\n- **输出**：在必要时调用 MCP 工具后生成的最终任务答案\n- **评分方式**：由 LLM 评判最终回答是否满足每条专家定义的声明\n\n## 核心特性\n\n- 使用 EvalScope 原生的 AgentLoop，而非 MCP-Atlas 自带的 `mcp_eval` 补全服务。\n- 直接连接 MCP-Atlas 的 `agent-environment` HTTP 服务，该服务必须暴露 `/enabled-servers`、`/list-tools` 和 `/call-tool` 端点。\n- 默认根据当前启用的 MCP 服务器过滤任务，以匹配 MCP-Atlas 公开脚本的行为（适用于未配置全部外部 API 密钥的环境）。\n- 仅向模型暴露任务指定的 `ENABLED_TOOLS`，避免一次性展示数百个工具。\n- 在同一样本内，若对 MCP 服务器的重复调用因传输层错误失败，则提前终止后续调用。\n- 报告平均 `coverage_score` 和 `pass_rate`，并支持配置通过阈值。\n\n## 评估说明\n\n- 运行此基准前，请先启动 MCP-Atlas 的 `agent-environment` Docker 服务，默认 URL 为 `http://localhost:1984`。\n- 此 EvalScope 原生适配器旨在便于在 EvalScope 内部维护。除非另行验证当前 Scale 排行榜所用配置，否则不保证与排行榜结果等效。\n- 要实现完整的公开数据集覆盖，需配置 MCP-Atlas 所需的外部 API 密钥和服务数据。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `mcp_atlas` |\n| **数据集ID** | [ScaleAI/MCP-Atlas](https://modelscope.cn/datasets/ScaleAI/MCP-Atlas/summary) |\n| **论文** | [Paper](https://static.scale.com/uploads/674f4cc7a74e35bcaae1c29a/MCP_Atlas.pdf) |\n| **标签** | `Agent`, `MultiTurn` |\n| **指标** | `coverage_score`, `pass` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `train` |\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 89 |\n| 提示词长度（平均） | 291.67 字符 |\n| 提示词长度（最小/最大） | 133 / 579 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"5dc7f8ae\",\n \"content\": \"I've been working on a local project for telling stories where I've already built a few react components. I want to check the required dependency version for the component that has the fewest lines of code.\"\n }\n ],\n \"target\": \"[\\\"['The component with the fewest lines of code is AspectRatio', 'The AspectRatio component uses `@radix-ui/react-aspect-ratio`', 'The required dependency version for `@radix-ui/react-aspect-ratio` is\\\\\\\\n \\\\\\\"^1.0.3\\\\\\\"']\\\"]\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"filesystem_read_multiple_files\",\n \"description\": \"Read the contents of multiple files simultaneously. This is more efficient than reading files one by one when you need to analyze or compare multiple files. Each file's content is returned with its path as a reference. Failed reads for individual files won't stop the entire operation. Only works within allowed directories.\",\n \"parameters\": {\n \"properties\": {\n \"paths\": {\n \"type\": \"array\",\n \"description\": \"Array of file paths to read. Each path must be a string pointing to a valid file within allowed directories.\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"paths\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"filesystem_directory_tree\",\n \"description\": \"Get a recursive tree view of files and directories as a JSON structure. Each entry includes 'name', 'type' (file/directory), and 'children' for directories. Files have no children array, while directories always have a children array (which may be empty). The output is formatted with 2-space indentation for readability. Only works within allowed directories.\",\n \"parameters\": {\n \"properties\": {\n \"path\": {\n \"type\": \"string\"\n },\n \"excludePatterns\": {\n \"type\": \"array\",\n \"default\": [],\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"filesystem_list_allowed_directories\",\n \"description\": \"Returns the list of directories that this server is allowed to access. Subdirectories within these allowed directories are also accessible. Use this to understand which directories and their nested paths are available before trying to access files.\",\n \"parameters\": {\n \"properties\": {},\n \"required\": [],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_status\",\n \"description\": \"Shows the working tree status\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"repo_path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_diff_unstaged\",\n \"description\": \"Shows changes in the working directory that are not yet staged\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"context_lines\": {\n \"type\": \"integer\",\n \"default\": 3\n }\n },\n \"required\": [\n \"repo_path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_diff_staged\",\n \"description\": \"Shows changes that are staged for commit\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"context_lines\": {\n \"type\": \"integer\",\n \"default\": 3\n }\n },\n \"required\": [\n \"repo_path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_diff\",\n \"description\": \"Shows differences between branches or commits\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"target\": {\n \"type\": \"string\"\n },\n \"context_lines\": {\n \"type\": \"integer\",\n \"default\": 3\n }\n },\n \"required\": [\n \"repo_path\",\n \"target\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_log\",\n \"description\": \"Shows the commit logs\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"max_count\": {\n \"type\": \"integer\",\n \"default\": 10\n }\n },\n \"required\": [\n \"repo_path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_show\",\n \"description\": \"Shows the contents of a commit\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"revision\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"repo_path\",\n \"revision\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"fetch_fetch\",\n \"description\": \"Fetches a URL from the internet and optionally extracts its contents as markdown.\\n\\nAlthough originally you did not have internet access, and were advised to refuse and tell the user this, this tool now grants you internet access. Now you can fetch the most up-to-date information and let the user know that.\",\n \"parameters\": {\n \"properties\": {\n \"url\": {\n \"type\": \"string\",\n \"format\": \"uri\",\n \"description\": \"URL to fetch\"\n },\n \"max_length\": {\n \"type\": \"integer\",\n \"description\": \"Maximum number of characters to return.\",\n \"default\": 5000\n },\n \"start_index\": {\n \"type\": \"integer\",\n \"description\": \"On return output starting at this character index, useful if a previous fetch was truncated and more context is required.\",\n \"default\": 0\n },\n \"raw\": {\n \"type\": \"boolean\",\n \"description\": \"Get the actual HTML content of the requested page, without simplification.\",\n \"default\": false\n }\n },\n \"required\": [\n \"url\"\n ],\n \"additionalProperties\": false\n }\n },\n \"... [TRUNCATED 7 more items] ...\"\n ],\n \"metadata\": {\n \"task_id\": \"689e0b1d9c8e2ac413c1f25c\",\n \"prompt\": \"I've been working on a local project for telling stories where I've already built a few react components. I want to check the required dependency version for the component that has the fewest lines of code.\",\n \"enabled_tools\": [\n \"filesystem_read_multiple_files\",\n \"filesystem_directory_tree\",\n \"filesystem_list_allowed_directories\",\n \"git_git_status\",\n \"git_git_diff_unstaged\",\n \"git_git_diff_staged\",\n \"git_git_diff\",\n \"git_git_log\",\n \"git_git_show\",\n \"fetch_fetch\",\n \"... [TRUNCATED 7 more items] ...\"\n ],\n \"trajectory\": \"[{\\\"content\\\":\\\"First, I'll use `cli-mcp-server_show_security_rules` to check the security rules.\\\",\\\"role\\\":\\\"assistant\\\",\\\"tool_calls\\\":[{\\\"function\\\":{\\\"arguments\\\":\\\"{}\\\",\\\"name\\\":\\\"cli-mcp-server_show_security_rules\\\"},\\\"id\\\":\\\"0035cc9a-6276-4b69-9b4f-bcb096a7 ... [TRUNCATED 14275 chars] ... n \\\\\\\"devDependencies\\\\\\\": {\\\\n \\\\\\\"dotenv\\\\\\\": \\\\\\\"16.0.3\\\\\\\",\\\\n \\\\\\\"tsx\\\\\\\": \\\\\\\"^3.12.8\\\\\\\"\\\\n }\\\\n}\\\\n\\\",\\\"type\\\":\\\"text\\\"},{\\\"text\\\":\\\"\\\\nCommand completed with return code: 0\\\",\\\"type\\\":\\\"text\\\"}],\\\"role\\\":\\\"tool\\\",\\\"tool_call_id\\\":\\\"6fb02c16-98f1-4a8d-843b-93d3f3f4d02c\\\"}]\",\n \"gtfa_claims\": [\n \"['The component with the fewest lines of code is AspectRatio', 'The AspectRatio component uses `@radix-ui/react-aspect-ratio`', 'The required dependency version for `@radix-ui/react-aspect-ratio` is\\\\n \\\"^1.0.3\\\"']\"\n ],\n \"required_servers\": [\n \"cli-mcp-server\",\n \"filesystem\"\n ],\n \"mcp_server_url\": \"http://localhost:1984\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `mcp_server_url` | `str` | `http://localhost:1984` | MCP-Atlas agent-environment 服务的基础 URL。 |\n| `filter_enabled_servers` | `bool` | `True` | 跳过那些其真实轨迹依赖当前未启用 MCP 服务器的任务。 |\n| `max_steps` | `int` | `100` | 每个样本允许的最大 EvalScope 智能体循环步数。 |\n| `max_tool_calls` | `int` | `100` | 每个样本允许的最大 MCP 工具调用次数。 |\n| `request_timeout` | `float` | `60.0` | MCP 工具调用的超时时间（秒）。 |\n| `list_tools_timeout` | `float` | `180.0` | MCP 服务器预检及 list-tools 请求的超时时间（秒）。 |\n| `use_system_prompt` | `bool` | `False` | 为每个样本前置 MCP-Atlas 可选的系统提示。 |\n| `pass_threshold` | `float` | `0.75` | 用于计算通过率的覆盖率得分阈值。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets mcp_atlas \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['mcp_atlas'],\n dataset_args={\n 'mcp_atlas': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "f7a6d5fc9fe66fbd229be0b43a38799b", + "en": "# MCP-Atlas\n\n## Overview\n\nMCP-Atlas is a Scale AI benchmark for evaluating tool-use competency with real Model Context Protocol\n(MCP) servers. It contains public tasks with prompts, allowed tool lists, ground-truth tool trajectories,\nand expert claims used for LLM-as-judge coverage scoring.\n\n## Task Description\n\n- **Task Type**: Tool-use agent benchmark\n- **Input**: Natural-language task prompt plus a per-task MCP tool allowlist\n- **Output**: Final task answer generated after using MCP tools when useful\n- **Grading**: LLM judge checks whether the final response fulfills each expert-defined claim\n\n## Key Features\n\n- Uses EvalScope's native AgentLoop rather than MCP-Atlas's `mcp_eval` completion service.\n- Connects directly to the MCP-Atlas `agent-environment` HTTP service, which must expose `/enabled-servers`,\n `/list-tools`, and `/call-tool`.\n- Filters tasks by currently enabled MCP servers by default, matching MCP-Atlas's public-script behavior for\n environments without every external API key configured.\n- Exposes only the task's `ENABLED_TOOLS` to the model to avoid advertising hundreds of tools at once.\n- Short-circuits repeated calls to MCP servers that hit transport-level failures inside the same sample.\n- Reports mean `coverage_score` and `pass_rate` with a configurable pass threshold.\n\n## Evaluation Notes\n\n- Start the MCP-Atlas `agent-environment` Docker service before running this benchmark. The default URL is\n `http://localhost:1984`.\n- This EvalScope-native adapter is intended to be maintainable inside EvalScope. It is not claimed to be\n leaderboard-equivalent unless the current Scale leaderboard harness settings are separately verified.\n- Full public-set coverage requires configuring the external API keys and service data required by MCP-Atlas.\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `mcp_atlas` |\n| **Dataset ID** | [ScaleAI/MCP-Atlas](https://modelscope.cn/datasets/ScaleAI/MCP-Atlas/summary) |\n| **Paper** | [Paper](https://static.scale.com/uploads/674f4cc7a74e35bcaae1c29a/MCP_Atlas.pdf) |\n| **Tags** | `Agent`, `MultiTurn` |\n| **Metrics** | `coverage_score`, `pass` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 89 |\n| Prompt Length (Mean) | 291.67 chars |\n| Prompt Length (Min/Max) | 133 / 579 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"5dc7f8ae\",\n \"content\": \"I've been working on a local project for telling stories where I've already built a few react components. I want to check the required dependency version for the component that has the fewest lines of code.\"\n }\n ],\n \"target\": \"[\\\"['The component with the fewest lines of code is AspectRatio', 'The AspectRatio component uses `@radix-ui/react-aspect-ratio`', 'The required dependency version for `@radix-ui/react-aspect-ratio` is\\\\\\\\n \\\\\\\"^1.0.3\\\\\\\"']\\\"]\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"filesystem_read_multiple_files\",\n \"description\": \"Read the contents of multiple files simultaneously. This is more efficient than reading files one by one when you need to analyze or compare multiple files. Each file's content is returned with its path as a reference. Failed reads for individual files won't stop the entire operation. Only works within allowed directories.\",\n \"parameters\": {\n \"properties\": {\n \"paths\": {\n \"type\": \"array\",\n \"description\": \"Array of file paths to read. Each path must be a string pointing to a valid file within allowed directories.\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"paths\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"filesystem_directory_tree\",\n \"description\": \"Get a recursive tree view of files and directories as a JSON structure. Each entry includes 'name', 'type' (file/directory), and 'children' for directories. Files have no children array, while directories always have a children array (which may be empty). The output is formatted with 2-space indentation for readability. Only works within allowed directories.\",\n \"parameters\": {\n \"properties\": {\n \"path\": {\n \"type\": \"string\"\n },\n \"excludePatterns\": {\n \"type\": \"array\",\n \"default\": [],\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"filesystem_list_allowed_directories\",\n \"description\": \"Returns the list of directories that this server is allowed to access. Subdirectories within these allowed directories are also accessible. Use this to understand which directories and their nested paths are available before trying to access files.\",\n \"parameters\": {\n \"properties\": {},\n \"required\": [],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_status\",\n \"description\": \"Shows the working tree status\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"repo_path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_diff_unstaged\",\n \"description\": \"Shows changes in the working directory that are not yet staged\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"context_lines\": {\n \"type\": \"integer\",\n \"default\": 3\n }\n },\n \"required\": [\n \"repo_path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_diff_staged\",\n \"description\": \"Shows changes that are staged for commit\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"context_lines\": {\n \"type\": \"integer\",\n \"default\": 3\n }\n },\n \"required\": [\n \"repo_path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_diff\",\n \"description\": \"Shows differences between branches or commits\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"target\": {\n \"type\": \"string\"\n },\n \"context_lines\": {\n \"type\": \"integer\",\n \"default\": 3\n }\n },\n \"required\": [\n \"repo_path\",\n \"target\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_log\",\n \"description\": \"Shows the commit logs\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"max_count\": {\n \"type\": \"integer\",\n \"default\": 10\n }\n },\n \"required\": [\n \"repo_path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_show\",\n \"description\": \"Shows the contents of a commit\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"revision\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"repo_path\",\n \"revision\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"fetch_fetch\",\n \"description\": \"Fetches a URL from the internet and optionally extracts its contents as markdown.\\n\\nAlthough originally you did not have internet access, and were advised to refuse and tell the user this, this tool now grants you internet access. Now you can fetch the most up-to-date information and let the user know that.\",\n \"parameters\": {\n \"properties\": {\n \"url\": {\n \"type\": \"string\",\n \"format\": \"uri\",\n \"description\": \"URL to fetch\"\n },\n \"max_length\": {\n \"type\": \"integer\",\n \"description\": \"Maximum number of characters to return.\",\n \"default\": 5000\n },\n \"start_index\": {\n \"type\": \"integer\",\n \"description\": \"On return output starting at this character index, useful if a previous fetch was truncated and more context is required.\",\n \"default\": 0\n },\n \"raw\": {\n \"type\": \"boolean\",\n \"description\": \"Get the actual HTML content of the requested page, without simplification.\",\n \"default\": false\n }\n },\n \"required\": [\n \"url\"\n ],\n \"additionalProperties\": false\n }\n },\n \"... [TRUNCATED 7 more items] ...\"\n ],\n \"metadata\": {\n \"task_id\": \"689e0b1d9c8e2ac413c1f25c\",\n \"prompt\": \"I've been working on a local project for telling stories where I've already built a few react components. I want to check the required dependency version for the component that has the fewest lines of code.\",\n \"enabled_tools\": [\n \"filesystem_read_multiple_files\",\n \"filesystem_directory_tree\",\n \"filesystem_list_allowed_directories\",\n \"git_git_status\",\n \"git_git_diff_unstaged\",\n \"git_git_diff_staged\",\n \"git_git_diff\",\n \"git_git_log\",\n \"git_git_show\",\n \"fetch_fetch\",\n \"... [TRUNCATED 7 more items] ...\"\n ],\n \"trajectory\": \"[{\\\"content\\\":\\\"First, I'll use `cli-mcp-server_show_security_rules` to check the security rules.\\\",\\\"role\\\":\\\"assistant\\\",\\\"tool_calls\\\":[{\\\"function\\\":{\\\"arguments\\\":\\\"{}\\\",\\\"name\\\":\\\"cli-mcp-server_show_security_rules\\\"},\\\"id\\\":\\\"0035cc9a-6276-4b69-9b4f-bcb096a7 ... [TRUNCATED 14275 chars] ... n \\\\\\\"devDependencies\\\\\\\": {\\\\n \\\\\\\"dotenv\\\\\\\": \\\\\\\"16.0.3\\\\\\\",\\\\n \\\\\\\"tsx\\\\\\\": \\\\\\\"^3.12.8\\\\\\\"\\\\n }\\\\n}\\\\n\\\",\\\"type\\\":\\\"text\\\"},{\\\"text\\\":\\\"\\\\nCommand completed with return code: 0\\\",\\\"type\\\":\\\"text\\\"}],\\\"role\\\":\\\"tool\\\",\\\"tool_call_id\\\":\\\"6fb02c16-98f1-4a8d-843b-93d3f3f4d02c\\\"}]\",\n \"gtfa_claims\": [\n \"['The component with the fewest lines of code is AspectRatio', 'The AspectRatio component uses `@radix-ui/react-aspect-ratio`', 'The required dependency version for `@radix-ui/react-aspect-ratio` is\\\\n \\\"^1.0.3\\\"']\"\n ],\n \"required_servers\": [\n \"cli-mcp-server\",\n \"filesystem\"\n ],\n \"mcp_server_url\": \"http://localhost:1984\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `mcp_server_url` | `str` | `http://localhost:1984` | MCP-Atlas agent-environment base URL. |\n| `filter_enabled_servers` | `bool` | `True` | Skip tasks whose ground-truth trajectory uses MCP servers that are not currently enabled. |\n| `max_tool_calls` | `int` | `100` | Maximum MCP tool calls allowed per sample. |\n| `request_timeout` | `float` | `60.0` | Timeout in seconds for MCP tool calls. |\n| `list_tools_timeout` | `float` | `180.0` | Timeout in seconds for MCP server preflight and list-tools requests. |\n| `use_system_prompt` | `bool` | `False` | Prepend the MCP-Atlas optional system prompt to every sample. |\n| `pass_threshold` | `float` | `0.75` | Coverage score threshold used to compute pass rate. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets mcp_atlas \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":100}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['mcp_atlas'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=100,\n ),\n dataset_args={\n 'mcp_atlas': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# MCP-Atlas\n\n## 概述\n\nMCP-Atlas 是 Scale AI 推出的一项基准测试，用于评估模型在真实 Model Context Protocol (MCP) 服务器环境下的工具使用能力。该基准包含公开任务，每个任务提供提示词、允许使用的工具列表、真实工具调用轨迹（ground-truth tool trajectories），以及用于 LLM-as-judge 覆盖率评分的专家声明（expert claims）。\n\n## 任务描述\n\n- **任务类型**：工具使用智能体基准测试\n- **输入**：自然语言任务提示词，以及每个任务对应的 MCP 工具白名单\n- **输出**：在必要时调用 MCP 工具后生成的最终任务答案\n- **评分方式**：由 LLM 评判最终回答是否满足每条专家定义的声明\n\n## 核心特性\n\n- 使用 EvalScope 原生的 AgentLoop，而非 MCP-Atlas 自带的 `mcp_eval` 补全服务。\n- 直接连接 MCP-Atlas 的 `agent-environment` HTTP 服务，该服务必须暴露 `/enabled-servers`、`/list-tools` 和 `/call-tool` 端点。\n- 默认根据当前启用的 MCP 服务器过滤任务，以匹配 MCP-Atlas 公开脚本在未配置全部外部 API 密钥环境下的行为。\n- 仅向模型暴露任务指定的 `ENABLED_TOOLS`，避免一次性展示数百个工具。\n- 在单个样本内，若对 MCP 服务器的重复调用因传输层错误失败，则提前终止后续调用。\n- 报告平均 `coverage_score` 和 `pass_rate`，并支持配置通过阈值。\n\n## 评测说明\n\n- 运行此基准前，请先启动 MCP-Atlas 的 `agent-environment` Docker 服务，默认 URL 为 `http://localhost:1984`。\n- 此 EvalScope 原生适配器旨在便于在 EvalScope 内部维护。除非单独验证当前 Scale 排行榜所用评测配置与此一致，否则不保证结果与排行榜等效。\n- 要实现完整的公开数据集覆盖，需配置 MCP-Atlas 所需的外部 API 密钥和服务数据。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `mcp_atlas` |\n| **数据集ID** | [ScaleAI/MCP-Atlas](https://modelscope.cn/datasets/ScaleAI/MCP-Atlas/summary) |\n| **论文** | [Paper](https://static.scale.com/uploads/674f4cc7a74e35bcaae1c29a/MCP_Atlas.pdf) |\n| **标签** | `Agent`, `MultiTurn` |\n| **指标** | `coverage_score`, `pass` |\n| **默认示例数** | 0-shot |\n| **评测划分** | `train` |\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 89 |\n| 提示词长度（平均） | 291.67 字符 |\n| 提示词长度（最小/最大） | 133 / 579 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"5dc7f8ae\",\n \"content\": \"I've been working on a local project for telling stories where I've already built a few react components. I want to check the required dependency version for the component that has the fewest lines of code.\"\n }\n ],\n \"target\": \"[\\\"['The component with the fewest lines of code is AspectRatio', 'The AspectRatio component uses `@radix-ui/react-aspect-ratio`', 'The required dependency version for `@radix-ui/react-aspect-ratio` is\\\\\\\\n \\\\\\\"^1.0.3\\\\\\\"']\\\"]\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"filesystem_read_multiple_files\",\n \"description\": \"Read the contents of multiple files simultaneously. This is more efficient than reading files one by one when you need to analyze or compare multiple files. Each file's content is returned with its path as a reference. Failed reads for individual files won't stop the entire operation. Only works within allowed directories.\",\n \"parameters\": {\n \"properties\": {\n \"paths\": {\n \"type\": \"array\",\n \"description\": \"Array of file paths to read. Each path must be a string pointing to a valid file within allowed directories.\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"paths\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"filesystem_directory_tree\",\n \"description\": \"Get a recursive tree view of files and directories as a JSON structure. Each entry includes 'name', 'type' (file/directory), and 'children' for directories. Files have no children array, while directories always have a children array (which may be empty). The output is formatted with 2-space indentation for readability. Only works within allowed directories.\",\n \"parameters\": {\n \"properties\": {\n \"path\": {\n \"type\": \"string\"\n },\n \"excludePatterns\": {\n \"type\": \"array\",\n \"default\": [],\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"filesystem_list_allowed_directories\",\n \"description\": \"Returns the list of directories that this server is allowed to access. Subdirectories within these allowed directories are also accessible. Use this to understand which directories and their nested paths are available before trying to access files.\",\n \"parameters\": {\n \"properties\": {},\n \"required\": [],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_status\",\n \"description\": \"Shows the working tree status\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"repo_path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_diff_unstaged\",\n \"description\": \"Shows changes in the working directory that are not yet staged\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"context_lines\": {\n \"type\": \"integer\",\n \"default\": 3\n }\n },\n \"required\": [\n \"repo_path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_diff_staged\",\n \"description\": \"Shows changes that are staged for commit\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"context_lines\": {\n \"type\": \"integer\",\n \"default\": 3\n }\n },\n \"required\": [\n \"repo_path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_diff\",\n \"description\": \"Shows differences between branches or commits\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"target\": {\n \"type\": \"string\"\n },\n \"context_lines\": {\n \"type\": \"integer\",\n \"default\": 3\n }\n },\n \"required\": [\n \"repo_path\",\n \"target\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_log\",\n \"description\": \"Shows the commit logs\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"max_count\": {\n \"type\": \"integer\",\n \"default\": 10\n }\n },\n \"required\": [\n \"repo_path\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"git_git_show\",\n \"description\": \"Shows the contents of a commit\",\n \"parameters\": {\n \"properties\": {\n \"repo_path\": {\n \"type\": \"string\"\n },\n \"revision\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"repo_path\",\n \"revision\"\n ],\n \"additionalProperties\": false\n }\n },\n {\n \"name\": \"fetch_fetch\",\n \"description\": \"Fetches a URL from the internet and optionally extracts its contents as markdown.\\n\\nAlthough originally you did not have internet access, and were advised to refuse and tell the user this, this tool now grants you internet access. Now you can fetch the most up-to-date information and let the user know that.\",\n \"parameters\": {\n \"properties\": {\n \"url\": {\n \"type\": \"string\",\n \"format\": \"uri\",\n \"description\": \"URL to fetch\"\n },\n \"max_length\": {\n \"type\": \"integer\",\n \"description\": \"Maximum number of characters to return.\",\n \"default\": 5000\n },\n \"start_index\": {\n \"type\": \"integer\",\n \"description\": \"On return output starting at this character index, useful if a previous fetch was truncated and more context is required.\",\n \"default\": 0\n },\n \"raw\": {\n \"type\": \"boolean\",\n \"description\": \"Get the actual HTML content of the requested page, without simplification.\",\n \"default\": false\n }\n },\n \"required\": [\n \"url\"\n ],\n \"additionalProperties\": false\n }\n },\n \"... [TRUNCATED 7 more items] ...\"\n ],\n \"metadata\": {\n \"task_id\": \"689e0b1d9c8e2ac413c1f25c\",\n \"prompt\": \"I've been working on a local project for telling stories where I've already built a few react components. I want to check the required dependency version for the component that has the fewest lines of code.\",\n \"enabled_tools\": [\n \"filesystem_read_multiple_files\",\n \"filesystem_directory_tree\",\n \"filesystem_list_allowed_directories\",\n \"git_git_status\",\n \"git_git_diff_unstaged\",\n \"git_git_diff_staged\",\n \"git_git_diff\",\n \"git_git_log\",\n \"git_git_show\",\n \"fetch_fetch\",\n \"... [TRUNCATED 7 more items] ...\"\n ],\n \"trajectory\": \"[{\\\"content\\\":\\\"First, I'll use `cli-mcp-server_show_security_rules` to check the security rules.\\\",\\\"role\\\":\\\"assistant\\\",\\\"tool_calls\\\":[{\\\"function\\\":{\\\"arguments\\\":\\\"{}\\\",\\\"name\\\":\\\"cli-mcp-server_show_security_rules\\\"},\\\"id\\\":\\\"0035cc9a-6276-4b69-9b4f-bcb096a7 ... [TRUNCATED 14275 chars] ... n \\\\\\\"devDependencies\\\\\\\": {\\\\n \\\\\\\"dotenv\\\\\\\": \\\\\\\"16.0.3\\\\\\\",\\\\n \\\\\\\"tsx\\\\\\\": \\\\\\\"^3.12.8\\\\\\\"\\\\n }\\\\n}\\\\n\\\",\\\"type\\\":\\\"text\\\"},{\\\"text\\\":\\\"\\\\nCommand completed with return code: 0\\\",\\\"type\\\":\\\"text\\\"}],\\\"role\\\":\\\"tool\\\",\\\"tool_call_id\\\":\\\"6fb02c16-98f1-4a8d-843b-93d3f3f4d02c\\\"}]\",\n \"gtfa_claims\": [\n \"['The component with the fewest lines of code is AspectRatio', 'The AspectRatio component uses `@radix-ui/react-aspect-ratio`', 'The required dependency version for `@radix-ui/react-aspect-ratio` is\\\\n \\\"^1.0.3\\\"']\"\n ],\n \"required_servers\": [\n \"cli-mcp-server\",\n \"filesystem\"\n ],\n \"mcp_server_url\": \"http://localhost:1984\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `mcp_server_url` | `str` | `http://localhost:1984` | MCP-Atlas agent-environment 服务的基础 URL。 |\n| `filter_enabled_servers` | `bool` | `True` | 跳过那些真实轨迹中使用了当前未启用 MCP 服务器的任务。 |\n| `max_tool_calls` | `int` | `100` | 每个样本允许的最大 MCP 工具调用次数。 |\n| `request_timeout` | `float` | `60.0` | MCP 工具调用的超时时间（秒）。 |\n| `list_tools_timeout` | `float` | `180.0` | MCP 服务器预检和 list-tools 请求的超时时间（秒）。 |\n| `use_system_prompt` | `bool` | `False` | 在每个样本前添加 MCP-Atlas 可选的系统提示。 |\n| `pass_threshold` | `float` | `0.75` | 用于计算通过率的覆盖率得分阈值。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets mcp_atlas \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":100}' \\\n --limit 10 # 正式评测时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['mcp_atlas'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=100,\n ),\n dataset_args={\n 'mcp_atlas': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评测时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "0e5695b85c692d3b10706dee807155d1", "needs_translation": false }, - "updated_at": "2026-06-30T17:13:44.559065", - "translation_updated_at": "2026-07-02T19:26:43" -} \ No newline at end of file + "updated_at": "2026-07-10T20:24:31.907075", + "translation_updated_at": "2026-07-10T20:24:42" +} diff --git a/evalscope/evalscope/benchmarks/_meta/measure_bench.json b/evalscope/evalscope/benchmarks/_meta/measure_bench.json new file mode 100644 index 0000000..10e221b --- /dev/null +++ b/evalscope/evalscope/benchmarks/_meta/measure_bench.json @@ -0,0 +1,200 @@ +{ + "meta": { + "pretty_name": "MeasureBench", + "dataset_id": "evalscope/MeasureBench", + "paper_url": "https://arxiv.org/abs/2510.26865", + "tags": [ + "MultiModal", + "Reasoning", + "QA" + ], + "metrics": [ + "acc", + "number_acc", + "unit_acc" + ], + "few_shot_num": 0, + "eval_split": "real_world", + "train_split": "", + "subset_list": [ + "real_world", + "synthetic_test" + ], + "description": "\n## Overview\n\nMeasureBench is a comprehensive benchmark for evaluating the ability of vision-language models (VLMs) to read values from measuring instruments. It covers both **real-world photographs** and **synthetically generated images** of 26 instrument types across 4 design categories.\n\n## Task Description\n\n- **Task Type**: Free-form Visual Question Answering (instrument reading)\n- **Input**: An image of a measuring instrument + a reading question\n- **Output**: The instrument's current reading (numeric value or time, with unit)\n- **Domains**: Ammeters, clocks, thermometers, scales, speedometers, and 21 more instrument types\n\n## Key Features\n\n- 2,442 total samples across two splits: real_world (1,272) and synthetic_test (1,170)\n- 26 instrument types, 4 design categories (dial, digital, analog, linear)\n- Accepts a tolerance interval around the correct value rather than requiring an exact match\n- For clocks: handles both 12-hour and 24-hour ambiguity via multiple valid intervals\n- Unit recognition is evaluated separately from numeric accuracy\n\n## Evaluation Notes\n\n- Default splits: **real_world** and **synthetic_test** (treated as separate subsets)\n- Primary metric: **Accuracy** (acc) — ``all_correct``: number *and* unit both correct\n- Secondary metrics: **number_acc** (numeric only), **unit_acc** (unit only)\n- Two evaluators: ``interval_matching`` (single valid range) and ``multi_interval_matching`` (e.g. clock AM/PM)\n- Model output is expected in the format ``Answer: `` on the last line\n- ``image_type`` is recorded in each sample's metadata; per-type results are visible in the\n ``subset_key`` column of review files but are not separately selectable via ``subset_list``\n- [Paper](https://arxiv.org/abs/2510.26865) | [GitHub](https://github.com/flageval-baai/MeasureBench)\n", + "prompt_template": "", + "system_prompt": "", + "few_shot_prompt_template": "", + "aggregation": "mean", + "extra_params": {}, + "sandbox_config": {}, + "category": "vlm" + }, + "statistics": { + "total_samples": 2442, + "subset_stats": [ + { + "name": "real_world", + "sample_count": 1272, + "prompt_length_mean": 153.83, + "prompt_length_min": 131, + "prompt_length_max": 215, + "prompt_length_std": 11.84, + "target_length_mean": null, + "multimodal": { + "has_images": true, + "has_audio": false, + "has_video": false, + "image": { + "count_total": 1272, + "count_per_sample": { + "min": 1, + "max": 1, + "mean": 1 + }, + "resolutions": [ + "1000x592", + "1000x845", + "1010x923", + "1012x841", + "1016x1024", + "1017x631", + "1018x1024", + "1022x1024", + "1024x1024", + "1024x312" + ], + "resolution_range": { + "min": "108x79", + "max": "3025x1599" + }, + "formats": [ + "jpeg", + "png" + ] + } + } + }, + { + "name": "synthetic_test", + "sample_count": 1170, + "prompt_length_mean": 147.71, + "prompt_length_min": 126, + "prompt_length_max": 192, + "prompt_length_std": 9.67, + "target_length_mean": null, + "multimodal": { + "has_images": true, + "has_audio": false, + "has_video": false, + "image": { + "count_total": 1170, + "count_per_sample": { + "min": 1, + "max": 1, + "mean": 1 + }, + "resolutions": [ + "1024x512", + "1080x472", + "1080x868", + "1100x420", + "1280x720", + "150x425", + "151x441", + "152x596", + "154x586", + "156x487" + ], + "resolution_range": { + "min": "419x150", + "max": "989x989" + }, + "formats": [ + "jpeg", + "png" + ] + } + } + } + ], + "prompt_length": { + "mean": 150.9, + "min": 126, + "max": 215, + "std": 11.27 + }, + "target_length_mean": null, + "computed_at": "2026-07-06T17:24:01.390403", + "multimodal": { + "has_images": true, + "has_audio": false, + "has_video": false, + "image": { + "count_total": 2442, + "count_per_sample": { + "min": 1, + "max": 1, + "mean": 1 + }, + "resolutions": [ + "1000x592", + "1000x845", + "1010x923", + "1012x841", + "1016x1024", + "1017x631", + "1018x1024", + "1022x1024", + "1024x1024", + "1024x312" + ], + "resolution_range": { + "min": "108x79", + "max": "3025x1599" + }, + "formats": [ + "jpeg", + "png" + ] + } + } + }, + "sample_example": { + "data": { + "input": [ + { + "id": "1341f508", + "content": [ + { + "image": "[BASE64_IMAGE: jpeg, ~75.8KB]" + }, + { + "text": "What is the reading of the instrument?\nProvide your final answer on the last line in the format: Answer: . For example: Answer: 42.5 A" + } + ] + } + ], + "target": "", + "id": 0, + "group_id": 0, + "subset_key": "ammeter", + "metadata": { + "question_id": "ammeter_0", + "image_type": "ammeter", + "design": "dial", + "evaluator": "interval_matching", + "evaluator_kwargs": "{\"interval\": [9.5, 9.7], \"units\": [\"A\", \"Ampere\"]}" + } + }, + "subset": "real_world", + "truncated": false + }, + "readme": { + "en": "# MeasureBench\n\n\n## Overview\n\nMeasureBench is a comprehensive benchmark for evaluating the ability of vision-language models (VLMs) to read values from measuring instruments. It covers both **real-world photographs** and **synthetically generated images** of 26 instrument types across 4 design categories.\n\n## Task Description\n\n- **Task Type**: Free-form Visual Question Answering (instrument reading)\n- **Input**: An image of a measuring instrument + a reading question\n- **Output**: The instrument's current reading (numeric value or time, with unit)\n- **Domains**: Ammeters, clocks, thermometers, scales, speedometers, and 21 more instrument types\n\n## Key Features\n\n- 2,442 total samples across two splits: real_world (1,272) and synthetic_test (1,170)\n- 26 instrument types, 4 design categories (dial, digital, analog, linear)\n- Accepts a tolerance interval around the correct value rather than requiring an exact match\n- For clocks: handles both 12-hour and 24-hour ambiguity via multiple valid intervals\n- Unit recognition is evaluated separately from numeric accuracy\n\n## Evaluation Notes\n\n- Default splits: **real_world** and **synthetic_test** (treated as separate subsets)\n- Primary metric: **Accuracy** (acc) — ``all_correct``: number *and* unit both correct\n- Secondary metrics: **number_acc** (numeric only), **unit_acc** (unit only)\n- Two evaluators: ``interval_matching`` (single valid range) and ``multi_interval_matching`` (e.g. clock AM/PM)\n- Model output is expected in the format ``Answer: `` on the last line\n- ``image_type`` is recorded in each sample's metadata; per-type results are visible in the\n ``subset_key`` column of review files but are not separately selectable via ``subset_list``\n- [Paper](https://arxiv.org/abs/2510.26865) | [GitHub](https://github.com/flageval-baai/MeasureBench)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `measure_bench` |\n| **Dataset ID** | [evalscope/MeasureBench](https://modelscope.cn/datasets/evalscope/MeasureBench/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2510.26865) |\n| **Tags** | `MultiModal`, `QA`, `Reasoning` |\n| **Metrics** | `acc`, `number_acc`, `unit_acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `real_world` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 2,442 |\n| Prompt Length (Mean) | 150.9 chars |\n| Prompt Length (Min/Max) | 126 / 215 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `real_world` | 1,272 | 153.83 | 131 | 215 |\n| `synthetic_test` | 1,170 | 147.71 | 126 | 192 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 2,442 |\n| Images per Sample | min: 1, max: 1, mean: 1 |\n| Resolution Range | 108x79 - 3025x1599 |\n| Formats | jpeg, png |\n\n\n## Sample Example\n\n**Subset**: `real_world`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"1341f508\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~75.8KB]\"\n },\n {\n \"text\": \"What is the reading of the instrument?\\nProvide your final answer on the last line in the format: Answer: . For example: Answer: 42.5 A\"\n }\n ]\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"ammeter\",\n \"metadata\": {\n \"question_id\": \"ammeter_0\",\n \"image_type\": \"ammeter\",\n \"design\": \"dial\",\n \"evaluator\": \"interval_matching\",\n \"evaluator_kwargs\": \"{\\\"interval\\\": [9.5, 9.7], \\\"units\\\": [\\\"A\\\", \\\"Ampere\\\"]}\"\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets measure_bench \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['measure_bench'],\n dataset_args={\n 'measure_bench': {\n # subset_list: ['real_world', 'synthetic_test'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# MeasureBench\n\n\n## 概述\n\nMeasureBench 是一个全面的基准测试，用于评估视觉-语言模型（VLMs）从测量仪器中读取数值的能力。该基准涵盖 **真实世界照片** 和 **合成生成图像**，包含 4 种设计类别下的 26 种仪器类型。\n\n## 任务描述\n\n- **任务类型**：开放式视觉问答（仪器读数）\n- **输入**：一张测量仪器的图像 + 一个读数问题\n- **输出**：仪器当前的读数（数值或时间，附带单位）\n- **领域**：电流表、时钟、温度计、秤、速度表等共 26 种仪器类型\n\n## 主要特性\n\n- 共计 2,442 个样本，分为两个子集：real_world（1,272）和 synthetic_test（1,170）\n- 包含 26 种仪器类型，4 种设计类别（指针式、数字式、模拟式、线性式）\n- 接受围绕正确值的一个容差区间，而非要求完全精确匹配\n- 对于时钟：通过多个有效区间处理 12 小时制与 24 小时制的歧义\n- 单位识别与数值准确性分别进行评估\n\n## 评估说明\n\n- 默认子集：**real_world** 和 **synthetic_test**（视为独立子集）\n- 主要指标：**Accuracy**（acc）—— ``all_correct``：数值 *和* 单位均正确\n- 次要指标：**number_acc**（仅数值）、**unit_acc**（仅单位）\n- 两种评估器：``interval_matching``（单一有效范围）和 ``multi_interval_matching``（例如时钟的上午/下午）\n- 模型输出应在最后一行以格式 ``Answer: `` 提供\n- 每个样本的元数据中记录了 ``image_type``；按类型的结果可在评审文件的 ``subset_key`` 列中查看，但无法通过 ``subset_list`` 单独选择\n- [论文](https://arxiv.org/abs/2510.26865) | [GitHub](https://github.com/flageval-baai/MeasureBench)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `measure_bench` |\n| **数据集ID** | [evalscope/MeasureBench](https://modelscope.cn/datasets/evalscope/MeasureBench/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2510.26865) |\n| **标签** | `MultiModal`, `QA`, `Reasoning` |\n| **指标** | `acc`, `number_acc`, `unit_acc` |\n| **默认示例数** | 0-shot |\n| **评估子集** | `real_world` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 2,442 |\n| 提示词长度（平均） | 150.9 字符 |\n| 提示词长度（最小/最大） | 126 / 215 字符 |\n\n**各子集统计：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `real_world` | 1,272 | 153.83 | 131 | 215 |\n| `synthetic_test` | 1,170 | 147.71 | 126 | 192 |\n\n**图像统计：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 2,442 |\n| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 |\n| 分辨率范围 | 108x79 - 3025x1599 |\n| 格式 | jpeg, png |\n\n\n## 样例示例\n\n**子集**: `real_world`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"1341f508\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~75.8KB]\"\n },\n {\n \"text\": \"What is the reading of the instrument?\\nProvide your final answer on the last line in the format: Answer: . For example: Answer: 42.5 A\"\n }\n ]\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"ammeter\",\n \"metadata\": {\n \"question_id\": \"ammeter_0\",\n \"image_type\": \"ammeter\",\n \"design\": \"dial\",\n \"evaluator\": \"interval_matching\",\n \"evaluator_kwargs\": \"{\\\"interval\\\": [9.5, 9.7], \\\"units\\\": [\\\"A\\\", \\\"Ampere\\\"]}\"\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets measure_bench \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['measure_bench'],\n dataset_args={\n 'measure_bench': {\n # subset_list: ['real_world', 'synthetic_test'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "18eea817090ab99000c150bbfe5c5fc6", + "needs_translation": false + }, + "updated_at": "2026-07-06T18:48:45.603152", + "translation_updated_at": "2026-07-06T18:49:04" +} diff --git a/evalscope/evalscope/benchmarks/_meta/med_mcqa.json b/evalscope/evalscope/benchmarks/_meta/med_mcqa.json index a3d2e38..204d5bf 100644 --- a/evalscope/evalscope/benchmarks/_meta/med_mcqa.json +++ b/evalscope/evalscope/benchmarks/_meta/med_mcqa.json @@ -77,4 +77,4 @@ }, "updated_at": "2026-01-28T17:31:32.294805", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mgsm.json b/evalscope/evalscope/benchmarks/_meta/mgsm.json index 63c971e..04cec3b 100644 --- a/evalscope/evalscope/benchmarks/_meta/mgsm.json +++ b/evalscope/evalscope/benchmarks/_meta/mgsm.json @@ -179,4 +179,4 @@ }, "updated_at": "2026-01-28T17:31:32.296114", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mia_bench.json b/evalscope/evalscope/benchmarks/_meta/mia_bench.json index 2b1e615..1c37786 100644 --- a/evalscope/evalscope/benchmarks/_meta/mia_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/mia_bench.json @@ -166,4 +166,4 @@ }, "updated_at": "2026-03-26T16:11:40.573971", "translation_updated_at": "2026-03-26T16:11:48Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/micro_vqa.json b/evalscope/evalscope/benchmarks/_meta/micro_vqa.json index 2648295..c69914c 100644 --- a/evalscope/evalscope/benchmarks/_meta/micro_vqa.json +++ b/evalscope/evalscope/benchmarks/_meta/micro_vqa.json @@ -192,4 +192,4 @@ }, "updated_at": "2026-01-28T17:31:32.296242", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/minerva_math.json b/evalscope/evalscope/benchmarks/_meta/minerva_math.json index bd93584..71cbe6b 100644 --- a/evalscope/evalscope/benchmarks/_meta/minerva_math.json +++ b/evalscope/evalscope/benchmarks/_meta/minerva_math.json @@ -78,4 +78,4 @@ }, "updated_at": "2026-01-28T17:31:32.296261", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/minimax_verifier.json b/evalscope/evalscope/benchmarks/_meta/minimax_verifier.json index ea5117e..f6bb369 100644 --- a/evalscope/evalscope/benchmarks/_meta/minimax_verifier.json +++ b/evalscope/evalscope/benchmarks/_meta/minimax_verifier.json @@ -84,4 +84,4 @@ }, "updated_at": "2026-05-26T22:52:30.516097", "translation_updated_at": "2026-05-26T22:52:34" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mit_movie_trivia.json b/evalscope/evalscope/benchmarks/_meta/mit_movie_trivia.json index d1a704e..767ae3d 100644 --- a/evalscope/evalscope/benchmarks/_meta/mit_movie_trivia.json +++ b/evalscope/evalscope/benchmarks/_meta/mit_movie_trivia.json @@ -113,4 +113,4 @@ }, "updated_at": "2026-01-28T17:31:32.364938", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mit_restaurant.json b/evalscope/evalscope/benchmarks/_meta/mit_restaurant.json index 8d5146a..836ec24 100644 --- a/evalscope/evalscope/benchmarks/_meta/mit_restaurant.json +++ b/evalscope/evalscope/benchmarks/_meta/mit_restaurant.json @@ -93,4 +93,4 @@ }, "updated_at": "2026-01-28T17:31:32.367847", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mm_bench.json b/evalscope/evalscope/benchmarks/_meta/mm_bench.json index a61b485..ead5951 100644 --- a/evalscope/evalscope/benchmarks/_meta/mm_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/mm_bench.json @@ -196,4 +196,4 @@ }, "updated_at": "2026-01-28T17:31:32.300321", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mm_star.json b/evalscope/evalscope/benchmarks/_meta/mm_star.json index f4b1b9b..c9aa1b0 100644 --- a/evalscope/evalscope/benchmarks/_meta/mm_star.json +++ b/evalscope/evalscope/benchmarks/_meta/mm_star.json @@ -367,4 +367,4 @@ }, "updated_at": "2026-01-28T17:31:32.303664", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mmau.json b/evalscope/evalscope/benchmarks/_meta/mmau.json index a8b0333..12a9a6b 100644 --- a/evalscope/evalscope/benchmarks/_meta/mmau.json +++ b/evalscope/evalscope/benchmarks/_meta/mmau.json @@ -47,4 +47,4 @@ }, "updated_at": "2026-06-23T15:36:34.409694+08:00", "translation_updated_at": "2026-06-23T15:50:44+08:00" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mmlu.json b/evalscope/evalscope/benchmarks/_meta/mmlu.json index 6493640..3041582 100644 --- a/evalscope/evalscope/benchmarks/_meta/mmlu.json +++ b/evalscope/evalscope/benchmarks/_meta/mmlu.json @@ -640,4 +640,4 @@ }, "updated_at": "2026-01-28T17:31:32.307451", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mmlu_pro.json b/evalscope/evalscope/benchmarks/_meta/mmlu_pro.json index 648e27e..1555bc5 100644 --- a/evalscope/evalscope/benchmarks/_meta/mmlu_pro.json +++ b/evalscope/evalscope/benchmarks/_meta/mmlu_pro.json @@ -216,4 +216,4 @@ }, "updated_at": "2026-01-28T17:31:32.309146", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mmlu_redux.json b/evalscope/evalscope/benchmarks/_meta/mmlu_redux.json index db8d915..a404efc 100644 --- a/evalscope/evalscope/benchmarks/_meta/mmlu_redux.json +++ b/evalscope/evalscope/benchmarks/_meta/mmlu_redux.json @@ -647,4 +647,4 @@ }, "updated_at": "2026-01-28T17:31:32.309800", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mmmlu.json b/evalscope/evalscope/benchmarks/_meta/mmmlu.json index 69e67f3..f08de4e 100644 --- a/evalscope/evalscope/benchmarks/_meta/mmmlu.json +++ b/evalscope/evalscope/benchmarks/_meta/mmmlu.json @@ -211,4 +211,4 @@ }, "updated_at": "2026-03-17T18:23:11.531727", "translation_updated_at": "2026-03-17T18:23:36Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mmmu.json b/evalscope/evalscope/benchmarks/_meta/mmmu.json index 0ad3fa2..58f4601 100644 --- a/evalscope/evalscope/benchmarks/_meta/mmmu.json +++ b/evalscope/evalscope/benchmarks/_meta/mmmu.json @@ -1377,4 +1377,4 @@ }, "updated_at": "2026-01-28T17:31:32.310849", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mmmu_pro.json b/evalscope/evalscope/benchmarks/_meta/mmmu_pro.json index de8bc12..5428ce4 100644 --- a/evalscope/evalscope/benchmarks/_meta/mmmu_pro.json +++ b/evalscope/evalscope/benchmarks/_meta/mmmu_pro.json @@ -1387,4 +1387,4 @@ }, "updated_at": "2026-01-28T17:31:32.311641", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mri_mcqa.json b/evalscope/evalscope/benchmarks/_meta/mri_mcqa.json index 7edc30b..15afeec 100644 --- a/evalscope/evalscope/benchmarks/_meta/mri_mcqa.json +++ b/evalscope/evalscope/benchmarks/_meta/mri_mcqa.json @@ -78,4 +78,4 @@ }, "updated_at": "2026-01-28T17:31:32.312287", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/msr_vtt.json b/evalscope/evalscope/benchmarks/_meta/msr_vtt.json index 259f37e..9be1b54 100644 --- a/evalscope/evalscope/benchmarks/_meta/msr_vtt.json +++ b/evalscope/evalscope/benchmarks/_meta/msr_vtt.json @@ -5,6 +5,7 @@ "paper_url": "https://www.microsoft.com/en-us/research/publication/msr-vtt-a-large-video-description-dataset-for-bridging-video-and-language/", "tags": [ "MultiModal", + "Video", "ImageCaptioning" ], "metrics": [ @@ -22,32 +23,12 @@ "subset_list": [ "default" ], - "description": "\n## Overview\n\nMSR-VTT is a large-scale open-domain video captioning benchmark for evaluating video-to-text generation.\nThe native adapter groups records by `video_id`, so multiple annotation rows for one video become one sample\nwith multiple reference captions.\n\n## Task Description\n\n- **Task Type**: Video captioning\n- **Input**: Video clip or URL\n- **Output**: One concise natural-language caption\n- **Domains**: Open-domain video understanding and description\n\n## Evaluation Notes\n\n- Default data source: `AI-ModelScope/msr-vtt` on ModelScope, `validation` split\n- Hugging Face `VLM2Vec/MSR-VTT` remains available by setting `extra_params.dataset_hub=\"huggingface\"`\n- Primary metric: **CIDEr**\n- Additional metrics: BLEU-1/2/3/4, METEOR, ROUGE-L\n- Set `extra_params.video_dir` to prefer local media files over URL metadata\n", + "description": "\n## Overview\n\nMSR-VTT is a large-scale open-domain video captioning benchmark for evaluating video-to-text generation.\nThe native adapter groups records by `video_id`, so multiple annotation rows for one video become one sample\nwith multiple reference captions.\n\n## Task Description\n\n- **Task Type**: Video captioning\n- **Input**: Video clip or URL\n- **Output**: One concise natural-language caption\n- **Domains**: Open-domain video understanding and description\n\n## Evaluation Notes\n\n- Default data source: `AI-ModelScope/msr-vtt` on ModelScope, `validation` split\n- Hugging Face `VLM2Vec/MSR-VTT` remains available by setting `dataset_hub=\"huggingface\"` in TaskConfig\n- Primary metric: **CIDEr**\n- Additional metrics: BLEU-1/2/3/4, METEOR, ROUGE-L\n- Set `extra_params.video_dir` to prefer local media files over URL metadata\n", "prompt_template": "Describe the video in one concise sentence.", "system_prompt": "", "few_shot_prompt_template": "", "aggregation": "mean", "extra_params": { - "dataset_hub": { - "type": "str", - "description": "Dataset hub used to load MSR-VTT annotations.", - "value": "modelscope", - "choices": [ - "huggingface", - "modelscope", - "local" - ] - }, - "eval_split": { - "type": "str", - "description": "Source split to load; defaults to validation for ModelScope and test for Hugging Face.", - "value": "" - }, - "dataset_revision": { - "type": "str", - "description": "Optional dataset revision; leave empty to use the hub default.", - "value": "" - }, "video_dir": { "type": "str", "description": "Optional local directory containing MSR-VTT video files.", @@ -160,11 +141,11 @@ "truncated": false }, "readme": { - "en": "# MSR-VTT\n\n\n## Overview\n\nMSR-VTT is a large-scale open-domain video captioning benchmark for evaluating video-to-text generation.\nThe native adapter groups records by `video_id`, so multiple annotation rows for one video become one sample\nwith multiple reference captions.\n\n## Task Description\n\n- **Task Type**: Video captioning\n- **Input**: Video clip or URL\n- **Output**: One concise natural-language caption\n- **Domains**: Open-domain video understanding and description\n\n## Evaluation Notes\n\n- Default data source: `AI-ModelScope/msr-vtt` on ModelScope, `validation` split\n- Hugging Face `VLM2Vec/MSR-VTT` remains available by setting `extra_params.dataset_hub=\"huggingface\"`\n- Primary metric: **CIDEr**\n- Additional metrics: BLEU-1/2/3/4, METEOR, ROUGE-L\n- Set `extra_params.video_dir` to prefer local media files over URL metadata\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `msr_vtt` |\n| **Dataset ID** | [AI-ModelScope/msr-vtt](https://modelscope.cn/datasets/AI-ModelScope/msr-vtt/summary) |\n| **Paper** | [Paper](https://www.microsoft.com/en-us/research/publication/msr-vtt-a-large-video-description-dataset-for-bridging-video-and-language/) |\n| **Tags** | `ImageCaptioning`, `MultiModal` |\n| **Metrics** | `Bleu_1`, `Bleu_2`, `Bleu_3`, `Bleu_4`, `METEOR`, `ROUGE_L`, `CIDEr` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `validation` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 497 |\n| Prompt Length (Mean) | 43 chars |\n| Prompt Length (Min/Max) | 43 / 43 chars |\n\n**Video Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Videos | 497 |\n| Videos per Sample | min: 1, max: 1, mean: 1 |\n| Formats | mp4 |\n\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"36044e4b\",\n \"content\": [\n {\n \"text\": \"Describe the video in one concise sentence.\"\n },\n {\n \"video\": \"https://www.youtube.com/watch?v=A9pM9iOuAzM\",\n \"format\": \"mp4\",\n \"start\": 116.03,\n \"end\": 126.21\n }\n ]\n }\n ],\n \"target\": \"[\\\"a family is having coversation\\\"]\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"references\": [\n \"a family is having coversation\"\n ],\n \"subset\": \"default\",\n \"dataset_id\": \"AI-ModelScope/msr-vtt\",\n \"dataset_hub\": \"modelscope\",\n \"video\": \"https://www.youtube.com/watch?v=A9pM9iOuAzM\",\n \"start\": 116.03,\n \"end\": 126.21,\n \"fps\": null,\n \"video_id\": \"video6513\",\n \"category\": 14\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\nDescribe the video in one concise sentence.\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `dataset_hub` | `str` | `modelscope` | Dataset hub used to load MSR-VTT annotations. Choices: ['huggingface', 'modelscope', 'local'] |\n| `eval_split` | `str` | `` | Source split to load; defaults to validation for ModelScope and test for Hugging Face. |\n| `dataset_revision` | `str` | `` | Optional dataset revision; leave empty to use the hub default. |\n| `video_dir` | `str` | `` | Optional local directory containing MSR-VTT video files. |\n| `video_extension` | `str` | `` | Optional extension override for local videos, for example \"mp4\". |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets msr_vtt \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['msr_vtt'],\n dataset_args={\n 'msr_vtt': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# MSR-VTT\n\n\n## 概述\n\nMSR-VTT 是一个大规模开放域视频描述（video captioning）基准测试，用于评估视频到文本的生成能力。 \n原生适配器按 `video_id` 对记录进行分组，因此同一视频的多条标注行会被合并为一个样本，并包含多个参考描述。\n\n## 任务描述\n\n- **任务类型**：视频描述（Video captioning）\n- **输入**：视频片段或 URL\n- **输出**：一条简洁的自然语言描述\n- **领域**：开放域视频理解与描述\n\n## 评估说明\n\n- 默认数据源：ModelScope 上的 `AI-ModelScope/msr-vtt`，使用 `validation` 划分\n- 通过设置 `extra_params.dataset_hub=\"huggingface\"`，仍可使用 Hugging Face 上的 `VLM2Vec/MSR-VTT`\n- 主要指标：**CIDEr**\n- 其他指标：BLEU-1/2/3/4、METEOR、ROUGE-L\n- 设置 `extra_params.video_dir` 可优先使用本地媒体文件而非 URL 元数据\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `msr_vtt` |\n| **数据集ID** | [AI-ModelScope/msr-vtt](https://modelscope.cn/datasets/AI-ModelScope/msr-vtt/summary) |\n| **论文** | [Paper](https://www.microsoft.com/en-us/research/publication/msr-vtt-a-large-video-description-dataset-for-bridging-video-and-language/) |\n| **标签** | `ImageCaptioning`, `MultiModal` |\n| **指标** | `Bleu_1`, `Bleu_2`, `Bleu_3`, `Bleu_4`, `METEOR`, `ROUGE_L`, `CIDEr` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `validation` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 497 |\n| 提示词长度（平均） | 43 字符 |\n| 提示词长度（最小/最大） | 43 / 43 字符 |\n\n**视频统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 视频总数 | 497 |\n| 每样本视频数 | 最小: 1, 最大: 1, 平均: 1 |\n| 格式 | mp4 |\n\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"36044e4b\",\n \"content\": [\n {\n \"text\": \"Describe the video in one concise sentence.\"\n },\n {\n \"video\": \"https://www.youtube.com/watch?v=A9pM9iOuAzM\",\n \"format\": \"mp4\",\n \"start\": 116.03,\n \"end\": 126.21\n }\n ]\n }\n ],\n \"target\": \"[\\\"a family is having coversation\\\"]\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"references\": [\n \"a family is having coversation\"\n ],\n \"subset\": \"default\",\n \"dataset_id\": \"AI-ModelScope/msr-vtt\",\n \"dataset_hub\": \"modelscope\",\n \"video\": \"https://www.youtube.com/watch?v=A9pM9iOuAzM\",\n \"start\": 116.03,\n \"end\": 126.21,\n \"fps\": null,\n \"video_id\": \"video6513\",\n \"category\": 14\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\nDescribe the video in one concise sentence.\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `dataset_hub` | `str` | `modelscope` | 用于加载 MSR-VTT 标注的数据集平台。选项：['huggingface', 'modelscope', 'local'] |\n| `eval_split` | `str` | `` | 要加载的数据划分；ModelScope 默认为 validation，Hugging Face 默认为 test。 |\n| `dataset_revision` | `str` | `` | 可选的数据集版本；留空则使用平台默认版本。 |\n| `video_dir` | `str` | `` | 包含 MSR-VTT 视频文件的本地目录（可选）。 |\n| `video_extension` | `str` | `` | 本地视频文件的扩展名覆盖项（可选），例如 \"mp4\"。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets msr_vtt \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['msr_vtt'],\n dataset_args={\n 'msr_vtt': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "b106d344a56143e3aff4b70587470309", + "en": "# MSR-VTT\n\n\n## Overview\n\nMSR-VTT is a large-scale open-domain video captioning benchmark for evaluating video-to-text generation.\nThe native adapter groups records by `video_id`, so multiple annotation rows for one video become one sample\nwith multiple reference captions.\n\n## Task Description\n\n- **Task Type**: Video captioning\n- **Input**: Video clip or URL\n- **Output**: One concise natural-language caption\n- **Domains**: Open-domain video understanding and description\n\n## Evaluation Notes\n\n- Default data source: `AI-ModelScope/msr-vtt` on ModelScope, `validation` split\n- Hugging Face `VLM2Vec/MSR-VTT` remains available by setting `dataset_hub=\"huggingface\"` in TaskConfig\n- Primary metric: **CIDEr**\n- Additional metrics: BLEU-1/2/3/4, METEOR, ROUGE-L\n- Set `extra_params.video_dir` to prefer local media files over URL metadata\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `msr_vtt` |\n| **Dataset ID** | [AI-ModelScope/msr-vtt](https://modelscope.cn/datasets/AI-ModelScope/msr-vtt/summary) |\n| **Paper** | [Paper](https://www.microsoft.com/en-us/research/publication/msr-vtt-a-large-video-description-dataset-for-bridging-video-and-language/) |\n| **Tags** | `ImageCaptioning`, `MultiModal`, `Video` |\n| **Metrics** | `Bleu_1`, `Bleu_2`, `Bleu_3`, `Bleu_4`, `METEOR`, `ROUGE_L`, `CIDEr` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `validation` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 497 |\n| Prompt Length (Mean) | 43 chars |\n| Prompt Length (Min/Max) | 43 / 43 chars |\n\n**Video Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Videos | 497 |\n| Videos per Sample | min: 1, max: 1, mean: 1 |\n| Formats | mp4 |\n\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"36044e4b\",\n \"content\": [\n {\n \"text\": \"Describe the video in one concise sentence.\"\n },\n {\n \"video\": \"https://www.youtube.com/watch?v=A9pM9iOuAzM\",\n \"format\": \"mp4\",\n \"start\": 116.03,\n \"end\": 126.21\n }\n ]\n }\n ],\n \"target\": \"[\\\"a family is having coversation\\\"]\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"references\": [\n \"a family is having coversation\"\n ],\n \"subset\": \"default\",\n \"dataset_id\": \"AI-ModelScope/msr-vtt\",\n \"dataset_hub\": \"modelscope\",\n \"video\": \"https://www.youtube.com/watch?v=A9pM9iOuAzM\",\n \"start\": 116.03,\n \"end\": 126.21,\n \"fps\": null,\n \"video_id\": \"video6513\",\n \"category\": 14\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\nDescribe the video in one concise sentence.\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `video_dir` | `str` | `` | Optional local directory containing MSR-VTT video files. |\n| `video_extension` | `str` | `` | Optional extension override for local videos, for example \"mp4\". |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets msr_vtt \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['msr_vtt'],\n dataset_args={\n 'msr_vtt': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# MSR-VTT\n\n\n## 概述\n\nMSR-VTT 是一个大规模开放域视频描述（video captioning）基准测试，用于评估视频到文本的生成能力。 \n原生适配器按 `video_id` 对记录进行分组，因此同一视频的多条标注行会被合并为一个样本，并包含多个参考描述。\n\n## 任务描述\n\n- **任务类型**：视频描述（Video captioning）\n- **输入**：视频片段或 URL\n- **输出**：一条简洁的自然语言描述\n- **领域**：开放域视频理解与描述\n\n## 评估说明\n\n- 默认数据源：ModelScope 上的 `AI-ModelScope/msr-vtt`，使用 `validation` 划分\n- 通过在 TaskConfig 中设置 `dataset_hub=\"huggingface\"`，仍可使用 Hugging Face 上的 `VLM2Vec/MSR-VTT`\n- 主要指标：**CIDEr**\n- 其他指标：BLEU-1/2/3/4、METEOR、ROUGE-L\n- 设置 `extra_params.video_dir` 可优先使用本地媒体文件而非 URL 元数据\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `msr_vtt` |\n| **数据集ID** | [AI-ModelScope/msr-vtt](https://modelscope.cn/datasets/AI-ModelScope/msr-vtt/summary) |\n| **论文** | [Paper](https://www.microsoft.com/en-us/research/publication/msr-vtt-a-large-video-description-dataset-for-bridging-video-and-language/) |\n| **标签** | `ImageCaptioning`, `MultiModal`, `Video` |\n| **指标** | `Bleu_1`, `Bleu_2`, `Bleu_3`, `Bleu_4`, `METEOR`, `ROUGE_L`, `CIDEr` |\n| **默认示例数量** | 0-shot |\n| **评估划分** | `validation` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 497 |\n| 提示词长度（平均） | 43 字符 |\n| 提示词长度（最小/最大） | 43 / 43 字符 |\n\n**视频统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 视频总数 | 497 |\n| 每样本视频数 | 最小: 1, 最大: 1, 平均: 1 |\n| 格式 | mp4 |\n\n\n## 样例示例\n\n**子集**：`default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"36044e4b\",\n \"content\": [\n {\n \"text\": \"Describe the video in one concise sentence.\"\n },\n {\n \"video\": \"https://www.youtube.com/watch?v=A9pM9iOuAzM\",\n \"format\": \"mp4\",\n \"start\": 116.03,\n \"end\": 126.21\n }\n ]\n }\n ],\n \"target\": \"[\\\"a family is having coversation\\\"]\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"references\": [\n \"a family is having coversation\"\n ],\n \"subset\": \"default\",\n \"dataset_id\": \"AI-ModelScope/msr-vtt\",\n \"dataset_hub\": \"modelscope\",\n \"video\": \"https://www.youtube.com/watch?v=A9pM9iOuAzM\",\n \"start\": 116.03,\n \"end\": 126.21,\n \"fps\": null,\n \"video_id\": \"video6513\",\n \"category\": 14\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\nDescribe the video in one concise sentence.\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `video_dir` | `str` | `` | 可选的本地目录，包含 MSR-VTT 视频文件。 |\n| `video_extension` | `str` | `` | 可选的本地视频扩展名覆盖，例如 \"mp4\"。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets msr_vtt \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['msr_vtt'],\n dataset_args={\n 'msr_vtt': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "d07c609aa5a9e1d708fe45b734d34c49", "needs_translation": false }, - "updated_at": "2026-06-09T14:22:02.215803", - "translation_updated_at": "2026-06-09T14:27:18" -} \ No newline at end of file + "updated_at": "2026-07-13T17:06:02.224578", + "translation_updated_at": "2026-07-13T17:06:26" +} diff --git a/evalscope/evalscope/benchmarks/_meta/msvd.json b/evalscope/evalscope/benchmarks/_meta/msvd.json index a27f9b1..d296bfb 100644 --- a/evalscope/evalscope/benchmarks/_meta/msvd.json +++ b/evalscope/evalscope/benchmarks/_meta/msvd.json @@ -5,6 +5,7 @@ "paper_url": "https://aclanthology.org/P11-1020/", "tags": [ "MultiModal", + "Video", "ImageCaptioning" ], "metrics": [ @@ -22,32 +23,12 @@ "subset_list": [ "default" ], - "description": "\n## Overview\n\nMSVD is a classic video captioning benchmark with short web videos annotated by many human captions.\nThe native adapter treats each video as one evaluation sample and uses all available captions as references.\n\n## Task Description\n\n- **Task Type**: Video captioning\n- **Input**: Video clip\n- **Output**: One concise natural-language caption\n- **Domains**: Open-domain video understanding and description\n\n## Evaluation Notes\n\n- Default data source: `evalscope/MSVD` on ModelScope, `test` split\n- Hugging Face `VLM2Vec/MSVD` remains available by setting `extra_params.dataset_hub=\"huggingface\"`\n- Primary metric: **CIDEr**\n- Additional metrics: BLEU-1/2/3/4, METEOR, ROUGE-L\n- Set `extra_params.video_dir` when the dataset only provides video file names and local media files are required\n", + "description": "\n## Overview\n\nMSVD is a classic video captioning benchmark with short web videos annotated by many human captions.\nThe native adapter treats each video as one evaluation sample and uses all available captions as references.\n\n## Task Description\n\n- **Task Type**: Video captioning\n- **Input**: Video clip\n- **Output**: One concise natural-language caption\n- **Domains**: Open-domain video understanding and description\n\n## Evaluation Notes\n\n- Default data source: `evalscope/MSVD` on ModelScope, `test` split\n- Hugging Face `VLM2Vec/MSVD` remains available by setting `dataset_hub=\"huggingface\"` in TaskConfig\n- Primary metric: **CIDEr**\n- Additional metrics: BLEU-1/2/3/4, METEOR, ROUGE-L\n- Set `extra_params.video_dir` when the dataset only provides video file names and local media files are required\n", "prompt_template": "Describe the video in one concise sentence.", "system_prompt": "", "few_shot_prompt_template": "", "aggregation": "mean", "extra_params": { - "dataset_hub": { - "type": "str", - "description": "Dataset hub used to load MSVD annotations.", - "value": "modelscope", - "choices": [ - "huggingface", - "modelscope", - "local" - ] - }, - "eval_split": { - "type": "str", - "description": "Source split to load; defaults to test.", - "value": "" - }, - "dataset_revision": { - "type": "str", - "description": "Optional dataset revision; leave empty to use the hub default.", - "value": "" - }, "video_dir": { "type": "str", "description": "Optional local directory containing MSVD video files.", @@ -165,11 +146,11 @@ "truncated": false }, "readme": { - "en": "# MSVD\n\n\n## Overview\n\nMSVD is a classic video captioning benchmark with short web videos annotated by many human captions.\nThe native adapter treats each video as one evaluation sample and uses all available captions as references.\n\n## Task Description\n\n- **Task Type**: Video captioning\n- **Input**: Video clip\n- **Output**: One concise natural-language caption\n- **Domains**: Open-domain video understanding and description\n\n## Evaluation Notes\n\n- Default data source: `evalscope/MSVD` on ModelScope, `test` split\n- Hugging Face `VLM2Vec/MSVD` remains available by setting `extra_params.dataset_hub=\"huggingface\"`\n- Primary metric: **CIDEr**\n- Additional metrics: BLEU-1/2/3/4, METEOR, ROUGE-L\n- Set `extra_params.video_dir` when the dataset only provides video file names and local media files are required\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `msvd` |\n| **Dataset ID** | [evalscope/MSVD](https://modelscope.cn/datasets/evalscope/MSVD/summary) |\n| **Paper** | [Paper](https://aclanthology.org/P11-1020/) |\n| **Tags** | `ImageCaptioning`, `MultiModal` |\n| **Metrics** | `Bleu_1`, `Bleu_2`, `Bleu_3`, `Bleu_4`, `METEOR`, `ROUGE_L`, `CIDEr` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 670 |\n| Prompt Length (Mean) | 43 chars |\n| Prompt Length (Min/Max) | 43 / 43 chars |\n\n**Video Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Videos | 670 |\n| Videos per Sample | min: 1, max: 1, mean: 1 |\n| Formats | mp4 |\n\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"a4b83275\",\n \"content\": [\n {\n \"text\": \"Describe the video in one concise sentence.\"\n },\n {\n \"video\": \"fr9H1WLcF1A_256_261.avi\",\n \"format\": \"mp4\"\n }\n ]\n }\n ],\n \"target\": \"[\\\"two young men are playing table tennis\\\", \\\"men are playing table tennis\\\", \\\"two men are playing table tennis\\\", \\\"two men are playing a tabletennis\\\", \\\"a couple of people are playing a game of ping pong\\\", \\\"peoples are playing table tennis\\\", \\\"two ... [TRUNCATED 306 chars] ... e boys are playing\\\", \\\"people are playing ping pong\\\", \\\"18 kids and counting show the newest baby josie\\\", \\\"there is some kids and playing to each other\\\", \\\"the men played pingpong together\\\", \\\"the boys are playing ping pong\\\", \\\"2 boys is playing\\\"]\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"references\": [\n \"two young men are playing table tennis\",\n \"men are playing table tennis\",\n \"two men are playing table tennis\",\n \"two men are playing a tabletennis\",\n \"a couple of people are playing a game of ping pong\",\n \"peoples are playing table tennis\",\n \"two men are playing pingpong\",\n \"two guys play table tennis\",\n \"two people are playing ping pong\",\n \"two boys are playing ping pong\",\n \"... [TRUNCATED 12 more items] ...\"\n ],\n \"subset\": \"default\",\n \"dataset_id\": \"evalscope/MSVD\",\n \"dataset_hub\": \"modelscope\",\n \"video\": \"fr9H1WLcF1A_256_261.avi\",\n \"video_id\": \"fr9H1WLcF1A_256_261\",\n \"source\": \"MSVD\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\nDescribe the video in one concise sentence.\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `dataset_hub` | `str` | `modelscope` | Dataset hub used to load MSVD annotations. Choices: ['huggingface', 'modelscope', 'local'] |\n| `eval_split` | `str` | `` | Source split to load; defaults to test. |\n| `dataset_revision` | `str` | `` | Optional dataset revision; leave empty to use the hub default. |\n| `video_dir` | `str` | `` | Optional local directory containing MSVD video files. |\n| `video_extension` | `str` | `` | Optional extension override for local videos, for example \"mp4\". |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets msvd \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['msvd'],\n dataset_args={\n 'msvd': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# MSVD\n\n\n## 概述\n\nMSVD 是一个经典的视频描述（video captioning）基准测试，包含大量带有多个人工标注字幕的短视频。\n原生适配器将每个视频视为一个评估样本，并使用所有可用字幕作为参考。\n\n## 任务描述\n\n- **任务类型**：视频描述（Video captioning）\n- **输入**：视频片段\n- **输出**：一条简洁的自然语言字幕\n- **领域**：开放域视频理解与描述\n\n## 评估说明\n\n- 默认数据源：ModelScope 上的 `evalscope/MSVD`，使用 `test` 划分\n- 通过设置 `extra_params.dataset_hub=\"huggingface\"`，仍可使用 Hugging Face 上的 `VLM2Vec/MSVD`\n- 主要指标：**CIDEr**\n- 其他指标：BLEU-1/2/3/4、METEOR、ROUGE-L\n- 当数据集仅提供视频文件名且需要本地媒体文件时，请设置 `extra_params.video_dir`\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `msvd` |\n| **数据集ID** | [evalscope/MSVD](https://modelscope.cn/datasets/evalscope/MSVD/summary) |\n| **论文** | [Paper](https://aclanthology.org/P11-1020/) |\n| **标签** | `ImageCaptioning`, `MultiModal` |\n| **指标** | `Bleu_1`, `Bleu_2`, `Bleu_3`, `Bleu_4`, `METEOR`, `ROUGE_L`, `CIDEr` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 670 |\n| 提示词长度（平均） | 43 字符 |\n| 提示词长度（最小/最大） | 43 / 43 字符 |\n\n**视频统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 视频总数 | 670 |\n| 每样本视频数 | 最小: 1, 最大: 1, 平均: 1 |\n| 格式 | mp4 |\n\n\n## 样例示例\n\n**子集**：`default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"a4b83275\",\n \"content\": [\n {\n \"text\": \"Describe the video in one concise sentence.\"\n },\n {\n \"video\": \"fr9H1WLcF1A_256_261.avi\",\n \"format\": \"mp4\"\n }\n ]\n }\n ],\n \"target\": \"[\\\"two young men are playing table tennis\\\", \\\"men are playing table tennis\\\", \\\"two men are playing table tennis\\\", \\\"two men are playing a tabletennis\\\", \\\"a couple of people are playing a game of ping pong\\\", \\\"peoples are playing table tennis\\\", \\\"two ... [TRUNCATED 306 chars] ... e boys are playing\\\", \\\"people are playing ping pong\\\", \\\"18 kids and counting show the newest baby josie\\\", \\\"there is some kids and playing to each other\\\", \\\"the men played pingpong together\\\", \\\"the boys are playing ping pong\\\", \\\"2 boys is playing\\\"]\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"references\": [\n \"two young men are playing table tennis\",\n \"men are playing table tennis\",\n \"two men are playing table tennis\",\n \"two men are playing a tabletennis\",\n \"a couple of people are playing a game of ping pong\",\n \"peoples are playing table tennis\",\n \"two men are playing pingpong\",\n \"two guys play table tennis\",\n \"two people are playing ping pong\",\n \"two boys are playing ping pong\",\n \"... [TRUNCATED 12 more items] ...\"\n ],\n \"subset\": \"default\",\n \"dataset_id\": \"evalscope/MSVD\",\n \"dataset_hub\": \"modelscope\",\n \"video\": \"fr9H1WLcF1A_256_261.avi\",\n \"video_id\": \"fr9H1WLcF1A_256_261\",\n \"source\": \"MSVD\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\nDescribe the video in one concise sentence.\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `dataset_hub` | `str` | `modelscope` | 用于加载 MSVD 标注的数据集平台。选项：['huggingface', 'modelscope', 'local'] |\n| `eval_split` | `str` | `` | 要加载的数据划分；默认为 test。 |\n| `dataset_revision` | `str` | `` | 可选的数据集版本；留空则使用平台默认版本。 |\n| `video_dir` | `str` | `` | 包含 MSVD 视频文件的本地目录（可选）。 |\n| `video_extension` | `str` | `` | 本地视频文件的扩展名覆盖项（可选），例如 \"mp4\"。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets msvd \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['msvd'],\n dataset_args={\n 'msvd': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "8b9a7bf6e44733c00672786b2b6afb04", + "en": "# MSVD\n\n\n## Overview\n\nMSVD is a classic video captioning benchmark with short web videos annotated by many human captions.\nThe native adapter treats each video as one evaluation sample and uses all available captions as references.\n\n## Task Description\n\n- **Task Type**: Video captioning\n- **Input**: Video clip\n- **Output**: One concise natural-language caption\n- **Domains**: Open-domain video understanding and description\n\n## Evaluation Notes\n\n- Default data source: `evalscope/MSVD` on ModelScope, `test` split\n- Hugging Face `VLM2Vec/MSVD` remains available by setting `dataset_hub=\"huggingface\"` in TaskConfig\n- Primary metric: **CIDEr**\n- Additional metrics: BLEU-1/2/3/4, METEOR, ROUGE-L\n- Set `extra_params.video_dir` when the dataset only provides video file names and local media files are required\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `msvd` |\n| **Dataset ID** | [evalscope/MSVD](https://modelscope.cn/datasets/evalscope/MSVD/summary) |\n| **Paper** | [Paper](https://aclanthology.org/P11-1020/) |\n| **Tags** | `ImageCaptioning`, `MultiModal`, `Video` |\n| **Metrics** | `Bleu_1`, `Bleu_2`, `Bleu_3`, `Bleu_4`, `METEOR`, `ROUGE_L`, `CIDEr` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 670 |\n| Prompt Length (Mean) | 43 chars |\n| Prompt Length (Min/Max) | 43 / 43 chars |\n\n**Video Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Videos | 670 |\n| Videos per Sample | min: 1, max: 1, mean: 1 |\n| Formats | mp4 |\n\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"a4b83275\",\n \"content\": [\n {\n \"text\": \"Describe the video in one concise sentence.\"\n },\n {\n \"video\": \"fr9H1WLcF1A_256_261.avi\",\n \"format\": \"mp4\"\n }\n ]\n }\n ],\n \"target\": \"[\\\"two young men are playing table tennis\\\", \\\"men are playing table tennis\\\", \\\"two men are playing table tennis\\\", \\\"two men are playing a tabletennis\\\", \\\"a couple of people are playing a game of ping pong\\\", \\\"peoples are playing table tennis\\\", \\\"two ... [TRUNCATED 306 chars] ... e boys are playing\\\", \\\"people are playing ping pong\\\", \\\"18 kids and counting show the newest baby josie\\\", \\\"there is some kids and playing to each other\\\", \\\"the men played pingpong together\\\", \\\"the boys are playing ping pong\\\", \\\"2 boys is playing\\\"]\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"references\": [\n \"two young men are playing table tennis\",\n \"men are playing table tennis\",\n \"two men are playing table tennis\",\n \"two men are playing a tabletennis\",\n \"a couple of people are playing a game of ping pong\",\n \"peoples are playing table tennis\",\n \"two men are playing pingpong\",\n \"two guys play table tennis\",\n \"two people are playing ping pong\",\n \"two boys are playing ping pong\",\n \"... [TRUNCATED 12 more items] ...\"\n ],\n \"subset\": \"default\",\n \"dataset_id\": \"evalscope/MSVD\",\n \"dataset_hub\": \"modelscope\",\n \"video\": \"fr9H1WLcF1A_256_261.avi\",\n \"video_id\": \"fr9H1WLcF1A_256_261\",\n \"source\": \"MSVD\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\nDescribe the video in one concise sentence.\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `video_dir` | `str` | `` | Optional local directory containing MSVD video files. |\n| `video_extension` | `str` | `` | Optional extension override for local videos, for example \"mp4\". |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets msvd \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['msvd'],\n dataset_args={\n 'msvd': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# MSVD\n\n\n## 概述\n\nMSVD 是一个经典的视频描述（video captioning）基准测试，包含大量带有多个人工标注字幕的短视频。\n原生适配器将每个视频视为一个评估样本，并使用所有可用字幕作为参考。\n\n## 任务描述\n\n- **任务类型**：视频描述（Video captioning）\n- **输入**：视频片段\n- **输出**：一句简洁的自然语言描述\n- **领域**：开放域视频理解与描述\n\n## 评估说明\n\n- 默认数据源：ModelScope 上的 `evalscope/MSVD`，使用 `test` 划分\n- 通过在 TaskConfig 中设置 `dataset_hub=\"huggingface\"`，仍可使用 Hugging Face 的 `VLM2Vec/MSVD`\n- 主要指标：**CIDEr**\n- 其他指标：BLEU-1/2/3/4、METEOR、ROUGE-L\n- 当数据集仅提供视频文件名且需要本地媒体文件时，请设置 `extra_params.video_dir`\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `msvd` |\n| **数据集ID** | [evalscope/MSVD](https://modelscope.cn/datasets/evalscope/MSVD/summary) |\n| **论文** | [Paper](https://aclanthology.org/P11-1020/) |\n| **标签** | `ImageCaptioning`, `MultiModal`, `Video` |\n| **指标** | `Bleu_1`, `Bleu_2`, `Bleu_3`, `Bleu_4`, `METEOR`, `ROUGE_L`, `CIDEr` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 670 |\n| 提示词长度（平均） | 43 字符 |\n| 提示词长度（最小/最大） | 43 / 43 字符 |\n\n**视频统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 总视频数 | 670 |\n| 每样本视频数 | 最小: 1, 最大: 1, 平均: 1 |\n| 格式 | mp4 |\n\n\n## 样例示例\n\n**子集**：`default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"a4b83275\",\n \"content\": [\n {\n \"text\": \"Describe the video in one concise sentence.\"\n },\n {\n \"video\": \"fr9H1WLcF1A_256_261.avi\",\n \"format\": \"mp4\"\n }\n ]\n }\n ],\n \"target\": \"[\\\"two young men are playing table tennis\\\", \\\"men are playing table tennis\\\", \\\"two men are playing table tennis\\\", \\\"two men are playing a tabletennis\\\", \\\"a couple of people are playing a game of ping pong\\\", \\\"peoples are playing table tennis\\\", \\\"two ... [TRUNCATED 306 chars] ... e boys are playing\\\", \\\"people are playing ping pong\\\", \\\"18 kids and counting show the newest baby josie\\\", \\\"there is some kids and playing to each other\\\", \\\"the men played pingpong together\\\", \\\"the boys are playing ping pong\\\", \\\"2 boys is playing\\\"]\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"references\": [\n \"two young men are playing table tennis\",\n \"men are playing table tennis\",\n \"two men are playing table tennis\",\n \"two men are playing a tabletennis\",\n \"a couple of people are playing a game of ping pong\",\n \"peoples are playing table tennis\",\n \"two men are playing pingpong\",\n \"two guys play table tennis\",\n \"two people are playing ping pong\",\n \"two boys are playing ping pong\",\n \"... [TRUNCATED 12 more items] ...\"\n ],\n \"subset\": \"default\",\n \"dataset_id\": \"evalscope/MSVD\",\n \"dataset_hub\": \"modelscope\",\n \"video\": \"fr9H1WLcF1A_256_261.avi\",\n \"video_id\": \"fr9H1WLcF1A_256_261\",\n \"source\": \"MSVD\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\nDescribe the video in one concise sentence.\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `video_dir` | `str` | `` | 可选，包含 MSVD 视频文件的本地目录。 |\n| `video_extension` | `str` | `` | 可选，用于覆盖本地视频的扩展名，例如 \"mp4\"。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets msvd \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['msvd'],\n dataset_args={\n 'msvd': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "341c9c7da8d9d294aa042b5eba9f8f95", "needs_translation": false }, - "updated_at": "2026-06-09T15:45:54.351306", - "translation_updated_at": "2026-06-09T15:46:01" -} \ No newline at end of file + "updated_at": "2026-07-13T17:06:02.225184", + "translation_updated_at": "2026-07-13T17:06:26" +} diff --git a/evalscope/evalscope/benchmarks/_meta/multi_if.json b/evalscope/evalscope/benchmarks/_meta/multi_if.json index 84cccfe..44446f3 100644 --- a/evalscope/evalscope/benchmarks/_meta/multi_if.json +++ b/evalscope/evalscope/benchmarks/_meta/multi_if.json @@ -175,4 +175,4 @@ }, "updated_at": "2026-01-28T17:31:41.521333", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/multi_nerd.json b/evalscope/evalscope/benchmarks/_meta/multi_nerd.json index 3e43edf..35fe7cb 100644 --- a/evalscope/evalscope/benchmarks/_meta/multi_nerd.json +++ b/evalscope/evalscope/benchmarks/_meta/multi_nerd.json @@ -129,4 +129,4 @@ }, "updated_at": "2026-01-28T17:31:32.367871", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/multiple_humaneval.json b/evalscope/evalscope/benchmarks/_meta/multiple_humaneval.json index ddfac8a..4c8efaf 100644 --- a/evalscope/evalscope/benchmarks/_meta/multiple_humaneval.json +++ b/evalscope/evalscope/benchmarks/_meta/multiple_humaneval.json @@ -257,4 +257,4 @@ }, "updated_at": "2026-05-15T16:15:11.828156", "translation_updated_at": "2026-05-15T16:15:18" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/multiple_mbpp.json b/evalscope/evalscope/benchmarks/_meta/multiple_mbpp.json index 83612b4..12f9cb3 100644 --- a/evalscope/evalscope/benchmarks/_meta/multiple_mbpp.json +++ b/evalscope/evalscope/benchmarks/_meta/multiple_mbpp.json @@ -257,4 +257,4 @@ }, "updated_at": "2026-05-15T16:15:11.829680", "translation_updated_at": "2026-05-15T16:15:18" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/music_trivia.json b/evalscope/evalscope/benchmarks/_meta/music_trivia.json index f6939a3..380e0ca 100644 --- a/evalscope/evalscope/benchmarks/_meta/music_trivia.json +++ b/evalscope/evalscope/benchmarks/_meta/music_trivia.json @@ -77,4 +77,4 @@ }, "updated_at": "2026-01-28T17:31:32.322167", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/musr.json b/evalscope/evalscope/benchmarks/_meta/musr.json index e865d71..cde8d31 100644 --- a/evalscope/evalscope/benchmarks/_meta/musr.json +++ b/evalscope/evalscope/benchmarks/_meta/musr.json @@ -94,4 +94,4 @@ }, "updated_at": "2026-01-28T17:31:32.324723", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/mvbench.json b/evalscope/evalscope/benchmarks/_meta/mvbench.json index 9f3dab5..6a2dcc9 100644 --- a/evalscope/evalscope/benchmarks/_meta/mvbench.json +++ b/evalscope/evalscope/benchmarks/_meta/mvbench.json @@ -5,6 +5,7 @@ "paper_url": "https://arxiv.org/abs/2311.17005", "tags": [ "MultiModal", + "Video", "MCQ" ], "metrics": [ @@ -21,28 +22,7 @@ "system_prompt": "", "few_shot_prompt_template": "", "aggregation": "mean", - "extra_params": { - "dataset_id": { - "type": "str", - "description": "Dataset repository ID or local dataset root for MVBench annotations and videos.", - "value": "PKU-Alignment/MVBench" - }, - "dataset_hub": { - "type": "str", - "description": "Dataset hub used to load annotations and video archives.", - "value": "modelscope", - "choices": [ - "huggingface", - "modelscope", - "local" - ] - }, - "dataset_revision": { - "type": "str", - "description": "Optional dataset revision; leave empty to use the hub default.", - "value": "" - } - }, + "extra_params": {}, "sandbox_config": {}, "category": "vlm" }, @@ -146,11 +126,11 @@ } }, "readme": { - "en": "# MVBench\n\n\n## Overview\n\nMVBench is a public multimodal video understanding benchmark covering temporal perception,\nattribute/state reasoning, symbolic ordering, and high-level cognition. This native adapter uses\nthe ModelScope `PKU-Alignment/MVBench` mirror by default, which provides JSON annotations plus\noptimized video archives.\n\n## Task Description\n\n- **Task Type**: Video multiple-choice question answering\n- **Input**: Video + question + answer choices\n- **Output**: Single correct answer letter\n- **Subsets**: 20 MVBench tasks; the default smoke-test subset is `action_antonym`\n\n## Evaluation Notes\n\n- Default configuration uses **0-shot** evaluation\n- Primary metric: **Accuracy**\n- The default `action_antonym` subset downloads a small public MP4 archive for quick validation\n- Full benchmark evaluation can be requested by setting `subset_list` to additional MVBench subsets\n- Time-bounded records keep start/end metadata and add a short segment instruction to the prompt\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `mvbench` |\n| **Dataset ID** | [PKU-Alignment/MVBench](https://modelscope.cn/datasets/PKU-Alignment/MVBench/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2311.17005) |\n| **Tags** | `MCQ`, `MultiModal` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 4,000 |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `action_antonym` | 200 | N/A | N/A | N/A |\n| `action_count` | 200 | N/A | N/A | N/A |\n| `action_localization` | 200 | N/A | N/A | N/A |\n| `action_prediction` | 200 | N/A | N/A | N/A |\n| `action_sequence` | 200 | N/A | N/A | N/A |\n| `character_order` | 200 | N/A | N/A | N/A |\n| `counterfactual_inference` | 200 | N/A | N/A | N/A |\n| `egocentric_navigation` | 200 | N/A | N/A | N/A |\n| `episodic_reasoning` | 200 | N/A | N/A | N/A |\n| `fine_grained_action` | 200 | N/A | N/A | N/A |\n| `fine_grained_pose` | 200 | N/A | N/A | N/A |\n| `moving_attribute` | 200 | N/A | N/A | N/A |\n| `moving_count` | 200 | N/A | N/A | N/A |\n| `moving_direction` | 200 | N/A | N/A | N/A |\n| `object_existence` | 200 | N/A | N/A | N/A |\n| `object_interaction` | 200 | N/A | N/A | N/A |\n| `object_shuffle` | 200 | N/A | N/A | N/A |\n| `scene_transition` | 200 | N/A | N/A | N/A |\n| `state_change` | 200 | N/A | N/A | N/A |\n| `unexpected_action` | 200 | N/A | N/A | N/A |\n\n## Sample Example\n\n*Sample example not available.*\n\n## Prompt Template\n\n**Prompt Template:**\n```text\nAnswer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of {letters}. Think step by step before answering.\n\n{question}\n\n{choices}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `dataset_id` | `str` | `PKU-Alignment/MVBench` | Dataset repository ID or local dataset root for MVBench annotations and videos. |\n| `dataset_hub` | `str` | `modelscope` | Dataset hub used to load annotations and video archives. Choices: ['huggingface', 'modelscope', 'local'] |\n| `dataset_revision` | `str` | `` | Optional dataset revision; leave empty to use the hub default. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets mvbench \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['mvbench'],\n dataset_args={\n 'mvbench': {\n # subset_list: ['action_antonym', 'action_count', 'action_localization'] # optional, evaluate specific subsets\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# MVBench\n\n\n## 概述\n\nMVBench 是一个公开的多模态视频理解基准测试，涵盖时间感知、属性/状态推理、符号排序和高级认知任务。此原生适配器默认使用 ModelScope 上的 `PKU-Alignment/MVBench` 镜像，该镜像提供 JSON 标注文件及优化后的视频压缩包。\n\n## 任务描述\n\n- **任务类型**：视频多项选择题问答（Video multiple-choice question answering）\n- **输入**：视频 + 问题 + 答案选项\n- **输出**：单个正确答案字母\n- **子集**：20 个 MVBench 任务；默认的冒烟测试子集为 `action_antonym`\n\n## 评估说明\n\n- 默认配置使用 **0-shot** 评估\n- 主要指标：**准确率（Accuracy）**\n- 默认的 `action_antonym` 子集会下载一个小型公开 MP4 压缩包用于快速验证\n- 可通过设置 `subset_list` 参数指定额外的 MVBench 子集以进行完整基准测试\n- 对于带时间范围的记录，保留起始/结束元数据，并在提示词中添加简短的片段指令\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `mvbench` |\n| **数据集ID** | [PKU-Alignment/MVBench](https://modelscope.cn/datasets/PKU-Alignment/MVBench/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2311.17005) |\n| **标签** | `MCQ`, `MultiModal` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `train` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 4,000 |\n\n**各子集统计信息：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `action_antonym` | 200 | N/A | N/A | N/A |\n| `action_count` | 200 | N/A | N/A | N/A |\n| `action_localization` | 200 | N/A | N/A | N/A |\n| `action_prediction` | 200 | N/A | N/A | N/A |\n| `action_sequence` | 200 | N/A | N/A | N/A |\n| `character_order` | 200 | N/A | N/A | N/A |\n| `counterfactual_inference` | 200 | N/A | N/A | N/A |\n| `egocentric_navigation` | 200 | N/A | N/A | N/A |\n| `episodic_reasoning` | 200 | N/A | N/A | N/A |\n| `fine_grained_action` | 200 | N/A | N/A | N/A |\n| `fine_grained_pose` | 200 | N/A | N/A | N/A |\n| `moving_attribute` | 200 | N/A | N/A | N/A |\n| `moving_count` | 200 | N/A | N/A | N/A |\n| `moving_direction` | 200 | N/A | N/A | N/A |\n| `object_existence` | 200 | N/A | N/A | N/A |\n| `object_interaction` | 200 | N/A | N/A | N/A |\n| `object_shuffle` | 200 | N/A | N/A | N/A |\n| `scene_transition` | 200 | N/A | N/A | N/A |\n| `state_change` | 200 | N/A | N/A | N/A |\n| `unexpected_action` | 200 | N/A | N/A | N/A |\n\n## 样例示例\n\n*样例示例不可用。*\n\n## 提示模板\n\n**提示模板：**\n```text\nAnswer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of {letters}. Think step by step before answering.\n\n{question}\n\n{choices}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `dataset_id` | `str` | `PKU-Alignment/MVBench` | MVBench 标注和视频的数据集仓库 ID 或本地数据集根目录。 |\n| `dataset_hub` | `str` | `modelscope` | 用于加载标注和视频压缩包的数据集平台。可选值：['huggingface', 'modelscope', 'local'] |\n| `dataset_revision` | `str` | `` | 可选的数据集版本；留空则使用平台默认版本。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets mvbench \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['mvbench'],\n dataset_args={\n 'mvbench': {\n # subset_list: ['action_antonym', 'action_count', 'action_localization'] # 可选，用于评估特定子集\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "900082497d6505ae8b9d07b4ef09a202", + "en": "# MVBench\n\n\n## Overview\n\nMVBench is a public multimodal video understanding benchmark covering temporal perception,\nattribute/state reasoning, symbolic ordering, and high-level cognition. This native adapter uses\nthe ModelScope `PKU-Alignment/MVBench` mirror by default, which provides JSON annotations plus\noptimized video archives.\n\n## Task Description\n\n- **Task Type**: Video multiple-choice question answering\n- **Input**: Video + question + answer choices\n- **Output**: Single correct answer letter\n- **Subsets**: 20 MVBench tasks; the default smoke-test subset is `action_antonym`\n\n## Evaluation Notes\n\n- Default configuration uses **0-shot** evaluation\n- Primary metric: **Accuracy**\n- The default `action_antonym` subset downloads a small public MP4 archive for quick validation\n- Full benchmark evaluation can be requested by setting `subset_list` to additional MVBench subsets\n- Time-bounded records keep start/end metadata and add a short segment instruction to the prompt\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `mvbench` |\n| **Dataset ID** | [PKU-Alignment/MVBench](https://modelscope.cn/datasets/PKU-Alignment/MVBench/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2311.17005) |\n| **Tags** | `MCQ`, `MultiModal`, `Video` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 4,000 |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `action_antonym` | 200 | N/A | N/A | N/A |\n| `action_count` | 200 | N/A | N/A | N/A |\n| `action_localization` | 200 | N/A | N/A | N/A |\n| `action_prediction` | 200 | N/A | N/A | N/A |\n| `action_sequence` | 200 | N/A | N/A | N/A |\n| `character_order` | 200 | N/A | N/A | N/A |\n| `counterfactual_inference` | 200 | N/A | N/A | N/A |\n| `egocentric_navigation` | 200 | N/A | N/A | N/A |\n| `episodic_reasoning` | 200 | N/A | N/A | N/A |\n| `fine_grained_action` | 200 | N/A | N/A | N/A |\n| `fine_grained_pose` | 200 | N/A | N/A | N/A |\n| `moving_attribute` | 200 | N/A | N/A | N/A |\n| `moving_count` | 200 | N/A | N/A | N/A |\n| `moving_direction` | 200 | N/A | N/A | N/A |\n| `object_existence` | 200 | N/A | N/A | N/A |\n| `object_interaction` | 200 | N/A | N/A | N/A |\n| `object_shuffle` | 200 | N/A | N/A | N/A |\n| `scene_transition` | 200 | N/A | N/A | N/A |\n| `state_change` | 200 | N/A | N/A | N/A |\n| `unexpected_action` | 200 | N/A | N/A | N/A |\n\n## Sample Example\n\n*Sample example not available.*\n\n## Prompt Template\n\n**Prompt Template:**\n```text\nAnswer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of {letters}. Think step by step before answering.\n\n{question}\n\n{choices}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets mvbench \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['mvbench'],\n dataset_args={\n 'mvbench': {\n # subset_list: ['action_antonym', 'action_count', 'action_localization'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# MVBench\n\n\n## 概述\n\nMVBench 是一个公开的多模态视频理解基准测试，涵盖时间感知、属性/状态推理、符号排序和高级认知任务。此原生适配器默认使用 ModelScope 上的 `PKU-Alignment/MVBench` 镜像，该镜像提供 JSON 标注以及优化后的视频压缩包。\n\n## 任务描述\n\n- **任务类型**：视频多项选择题问答（Video multiple-choice question answering）\n- **输入**：视频 + 问题 + 答案选项\n- **输出**：单个正确答案字母\n- **子集**：20 个 MVBench 任务；默认的冒烟测试子集为 `action_antonym`\n\n## 评估说明\n\n- 默认配置使用 **0-shot** 评估\n- 主要指标：**准确率（Accuracy）**\n- 默认的 `action_antonym` 子集会下载一个小型公开 MP4 压缩包用于快速验证\n- 可通过设置 `subset_list` 参数指定额外的 MVBench 子集以进行完整基准评估\n- 对于带时间范围的记录，保留起始/结束时间元数据，并在提示词中添加简短的片段指令\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `mvbench` |\n| **数据集ID** | [PKU-Alignment/MVBench](https://modelscope.cn/datasets/PKU-Alignment/MVBench/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2311.17005) |\n| **标签** | `MCQ`, `MultiModal`, `Video` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `train` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 4,000 |\n\n**各子集统计数据：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `action_antonym` | 200 | N/A | N/A | N/A |\n| `action_count` | 200 | N/A | N/A | N/A |\n| `action_localization` | 200 | N/A | N/A | N/A |\n| `action_prediction` | 200 | N/A | N/A | N/A |\n| `action_sequence` | 200 | N/A | N/A | N/A |\n| `character_order` | 200 | N/A | N/A | N/A |\n| `counterfactual_inference` | 200 | N/A | N/A | N/A |\n| `egocentric_navigation` | 200 | N/A | N/A | N/A |\n| `episodic_reasoning` | 200 | N/A | N/A | N/A |\n| `fine_grained_action` | 200 | N/A | N/A | N/A |\n| `fine_grained_pose` | 200 | N/A | N/A | N/A |\n| `moving_attribute` | 200 | N/A | N/A | N/A |\n| `moving_count` | 200 | N/A | N/A | N/A |\n| `moving_direction` | 200 | N/A | N/A | N/A |\n| `object_existence` | 200 | N/A | N/A | N/A |\n| `object_interaction` | 200 | N/A | N/A | N/A |\n| `object_shuffle` | 200 | N/A | N/A | N/A |\n| `scene_transition` | 200 | N/A | N/A | N/A |\n| `state_change` | 200 | N/A | N/A | N/A |\n| `unexpected_action` | 200 | N/A | N/A | N/A |\n\n## 样例示例\n\n*样例示例不可用。*\n\n## 提示模板\n\n**提示模板：**\n```text\n请回答以下多项选择题。你的回复最后一行应采用如下格式：'ANSWER: [LETTER]'（不含引号），其中 [LETTER] 是 {letters} 中的一个字母。请逐步思考后再作答。\n\n{question}\n\n{choices}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets mvbench \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['mvbench'],\n dataset_args={\n 'mvbench': {\n # subset_list: ['action_antonym', 'action_count', 'action_localization'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "1907fd8d76489eaf79db3488a8bd3b6c", "needs_translation": false }, - "updated_at": "2026-05-18T10:23:17.450629", - "translation_updated_at": "2026-05-18T10:23:22" -} \ No newline at end of file + "updated_at": "2026-07-13T17:06:02.257616", + "translation_updated_at": "2026-07-13T17:06:26" +} diff --git a/evalscope/evalscope/benchmarks/_meta/ncbi.json b/evalscope/evalscope/benchmarks/_meta/ncbi.json index 514e0aa..f98d4cb 100644 --- a/evalscope/evalscope/benchmarks/_meta/ncbi.json +++ b/evalscope/evalscope/benchmarks/_meta/ncbi.json @@ -115,4 +115,4 @@ }, "updated_at": "2026-01-28T17:31:32.372008", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/needle_haystack.json b/evalscope/evalscope/benchmarks/_meta/needle_haystack.json index d0b22f8..b2a9195 100644 --- a/evalscope/evalscope/benchmarks/_meta/needle_haystack.json +++ b/evalscope/evalscope/benchmarks/_meta/needle_haystack.json @@ -142,4 +142,4 @@ }, "updated_at": "2026-01-28T17:31:32.325571", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/ocr_bench.json b/evalscope/evalscope/benchmarks/_meta/ocr_bench.json index 7c0b5ad..f436959 100644 --- a/evalscope/evalscope/benchmarks/_meta/ocr_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/ocr_bench.json @@ -525,4 +525,4 @@ }, "updated_at": "2026-01-28T17:31:32.380405", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/ocr_bench_v2.json b/evalscope/evalscope/benchmarks/_meta/ocr_bench_v2.json index fc041f4..397a499 100644 --- a/evalscope/evalscope/benchmarks/_meta/ocr_bench_v2.json +++ b/evalscope/evalscope/benchmarks/_meta/ocr_bench_v2.json @@ -1360,4 +1360,4 @@ }, "updated_at": "2026-01-28T17:31:32.424696", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/officeqa.json b/evalscope/evalscope/benchmarks/_meta/officeqa.json new file mode 100644 index 0000000..9e3558a --- /dev/null +++ b/evalscope/evalscope/benchmarks/_meta/officeqa.json @@ -0,0 +1,83 @@ +{ + "meta": { + "pretty_name": "OfficeQA", + "dataset_id": "evalscope/officeqa", + "paper_url": null, + "tags": [ + "Agent", + "QA", + "Knowledge" + ], + "metrics": [ + "acc" + ], + "few_shot_num": 0, + "eval_split": "train", + "train_split": "", + "subset_list": [ + "officeqa_pro" + ], + "description": "\n## Overview\n\nOfficeQA is a grounded reasoning benchmark by Databricks, built for evaluating model/agent performance on end-to-end grounded reasoning tasks over U.S. Treasury Bulletin documents (1939-2025).\n\n## Task Description\n\n- **Task Type**: Agent-based Document QA (grep/search over corpus)\n- **Input**: A question + access to parsed Treasury Bulletin text files via bash tools\n- **Output**: A precise answer (numeric values, text, or structured data)\n- **Evaluation Mode**: Agent with bash tool (grep, cat, etc.) over the corpus\n\n## Key Features\n\n- Two subsets: `officeqa_pro` (133 questions, hard, default) and `officeqa_full` (246 questions, easy+hard)\n- Corpus: ~900 parsed Treasury Bulletin text files (~460MB total)\n- Agent uses bash tools (grep, cat, head, etc.) to search the corpus\n- Scoring uses fuzzy numeric matching with configurable tolerance (1% default)\n\n## Evaluation Notes\n\n- The agent is given access to parsed .txt files in a corpus directory\n- Each question's `source_files` field indicates which document(s) contain the answer\n- Uses **rule-based scoring** adapted from official reward.py\n- Numerical answers matched with 1% relative error tolerance\n- Text answers use case-insensitive substring matching\n", + "prompt_template": "{question}", + "system_prompt": "", + "few_shot_prompt_template": "", + "aggregation": "mean", + "extra_params": {}, + "sandbox_config": {}, + "agent_config": { + "strategy": "function_calling", + "max_steps": 15 + }, + "category": "agent" + }, + "statistics": { + "total_samples": 133, + "subset_stats": [ + { + "name": "default", + "sample_count": 133, + "prompt_length_mean": 443.06, + "prompt_length_min": 165, + "prompt_length_max": 1186, + "prompt_length_std": 189.83, + "target_length_mean": 7.84 + } + ], + "prompt_length": { + "mean": 443.06, + "min": 165, + "max": 1186, + "std": 189.83 + }, + "target_length_mean": 7.84, + "computed_at": "2026-07-03T17:22:52.758222" + }, + "sample_example": { + "data": { + "input": [ + { + "id": "a6357de6", + "content": "What were the total expenditures (in millions of nominal dollars) for U.S national defense in the calendar year of 1940?\nPlease provide a precise and concise answer." + } + ], + "target": "2,602", + "id": 0, + "group_id": 0, + "metadata": { + "uid": "UID0001", + "source_files": "treasury_bulletin_1941_01.txt", + "difficulty": "hard" + } + }, + "subset": "default", + "truncated": false + }, + "readme": { + "en": "# OfficeQA\n\n\n## Overview\n\nOfficeQA is a grounded reasoning benchmark by Databricks, built for evaluating model/agent performance on end-to-end grounded reasoning tasks over U.S. Treasury Bulletin documents (1939-2025).\n\n## Task Description\n\n- **Task Type**: Agent-based Document QA (grep/search over corpus)\n- **Input**: A question + access to parsed Treasury Bulletin text files via bash tools\n- **Output**: A precise answer (numeric values, text, or structured data)\n- **Evaluation Mode**: Agent with bash tool (grep, cat, etc.) over the corpus\n\n## Key Features\n\n- Two subsets: `officeqa_pro` (133 questions, hard, default) and `officeqa_full` (246 questions, easy+hard)\n- Corpus: ~900 parsed Treasury Bulletin text files (~460MB total)\n- Agent uses bash tools (grep, cat, head, etc.) to search the corpus\n- Scoring uses fuzzy numeric matching with configurable tolerance (1% default)\n\n## Evaluation Notes\n\n- The agent is given access to parsed .txt files in a corpus directory\n- Each question's `source_files` field indicates which document(s) contain the answer\n- Uses **rule-based scoring** adapted from official reward.py\n- Numerical answers matched with 1% relative error tolerance\n- Text answers use case-insensitive substring matching\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `officeqa` |\n| **Dataset ID** | [evalscope/officeqa](https://modelscope.cn/datasets/evalscope/officeqa/summary) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `Knowledge`, `QA` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 133 |\n| Prompt Length (Mean) | 443.06 chars |\n| Prompt Length (Min/Max) | 165 / 1186 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"a6357de6\",\n \"content\": \"What were the total expenditures (in millions of nominal dollars) for U.S national defense in the calendar year of 1940?\\nPlease provide a precise and concise answer.\"\n }\n ],\n \"target\": \"2,602\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"uid\": \"UID0001\",\n \"source_files\": \"treasury_bulletin_1941_01.txt\",\n \"difficulty\": \"hard\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets officeqa \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":15}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['officeqa'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=15,\n ),\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# OfficeQA\n\n\n## 概述\n\nOfficeQA 是由 Databricks 构建的一个基于真实文档的推理基准测试，用于评估模型/智能体在 1939–2025 年美国财政部公告（U.S. Treasury Bulletin）文档上执行端到端 grounded reasoning 任务的性能。\n\n## 任务描述\n\n- **任务类型**：基于智能体的文档问答（通过 grep/search 在语料库中检索）\n- **输入**：一个问题 + 通过 bash 工具访问已解析的财政部公告文本文件\n- **输出**：精确答案（数值、文本或结构化数据）\n- **评估模式**：智能体使用 bash 工具（如 grep、cat 等）在语料库上进行检索\n\n## 主要特性\n\n- 包含两个子集：`officeqa_pro`（133 个问题，难度高，默认使用）和 `officeqa_full`（246 个问题，包含简单与困难问题）\n- 语料库：约 900 个已解析的财政部公告文本文件（总计约 460MB）\n- 智能体使用 bash 工具（grep、cat、head 等）搜索语料库\n- 评分采用模糊数值匹配，支持可配置容差（默认为 1%）\n\n## 评估说明\n\n- 智能体可访问语料库目录中的已解析 .txt 文件\n- 每个问题的 `source_files` 字段指明了包含答案的文档\n- 使用从官方 reward.py 改编的**基于规则的评分机制**\n- 数值答案允许 1% 的相对误差容差\n- 文本答案采用不区分大小写的子串匹配\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `officeqa` |\n| **数据集ID** | [evalscope/officeqa](https://modelscope.cn/datasets/evalscope/officeqa/summary) |\n| **论文** | N/A |\n| **标签** | `Agent`, `Knowledge`, `QA` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `train` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 133 |\n| 提示词长度（平均） | 443.06 字符 |\n| 提示词长度（最小/最大） | 165 / 1186 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"a6357de6\",\n \"content\": \"What were the total expenditures (in millions of nominal dollars) for U.S national defense in the calendar year of 1940?\\nPlease provide a precise and concise answer.\"\n }\n ],\n \"target\": \"2,602\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"uid\": \"UID0001\",\n \"source_files\": \"treasury_bulletin_1941_01.txt\",\n \"difficulty\": \"hard\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板:**\n```text\n{question}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets officeqa \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":15}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['officeqa'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=15,\n ),\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "29c72438a8d802f0c207a86ba2c03a03", + "needs_translation": false + }, + "updated_at": "2026-07-10T20:24:31.935621", + "translation_updated_at": "2026-07-10T20:24:42" +} diff --git a/evalscope/evalscope/benchmarks/_meta/olympiad_bench.json b/evalscope/evalscope/benchmarks/_meta/olympiad_bench.json index d4c1a24..95b2c4b 100644 --- a/evalscope/evalscope/benchmarks/_meta/olympiad_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/olympiad_bench.json @@ -590,4 +590,4 @@ }, "updated_at": "2026-01-28T17:31:32.393195", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/omni_bench.json b/evalscope/evalscope/benchmarks/_meta/omni_bench.json index e0350df..269aedb 100644 --- a/evalscope/evalscope/benchmarks/_meta/omni_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/omni_bench.json @@ -199,4 +199,4 @@ }, "updated_at": "2026-01-28T17:31:32.394498", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/omni_doc_bench.json b/evalscope/evalscope/benchmarks/_meta/omni_doc_bench.json index 9a9c64b..b32e493 100644 --- a/evalscope/evalscope/benchmarks/_meta/omni_doc_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/omni_doc_bench.json @@ -637,4 +637,4 @@ }, "updated_at": "2026-01-28T17:31:32.657409", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/ontonotes5.json b/evalscope/evalscope/benchmarks/_meta/ontonotes5.json index 22177e2..608d217 100644 --- a/evalscope/evalscope/benchmarks/_meta/ontonotes5.json +++ b/evalscope/evalscope/benchmarks/_meta/ontonotes5.json @@ -143,4 +143,4 @@ }, "updated_at": "2026-01-28T17:31:32.371262", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/openai_mrcr.json b/evalscope/evalscope/benchmarks/_meta/openai_mrcr.json index fed4b9e..f85ed32 100644 --- a/evalscope/evalscope/benchmarks/_meta/openai_mrcr.json +++ b/evalscope/evalscope/benchmarks/_meta/openai_mrcr.json @@ -142,4 +142,4 @@ }, "updated_at": "2026-01-28T17:59:01.303799", "translation_updated_at": "2026-01-28T18:07:35Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/perspective_gap_prompt_writing.json b/evalscope/evalscope/benchmarks/_meta/perspective_gap_prompt_writing.json new file mode 100644 index 0000000..51c097a --- /dev/null +++ b/evalscope/evalscope/benchmarks/_meta/perspective_gap_prompt_writing.json @@ -0,0 +1,138 @@ +{ + "meta": { + "pretty_name": "PerspectiveGap Prompt Writing", + "dataset_id": "evalscope/PerspectiveGap", + "paper_url": "https://arxiv.org/abs/2606.08878", + "tags": [ + "Agent", + "InstructionFollowing" + ], + "metrics": [ + "strict_pass", + "net_match_score", + "required_coverage", + "boundary_precision", + "distractor_leakage" + ], + "few_shot_num": 0, + "eval_split": "test", + "train_split": "", + "subset_list": [ + "default" + ], + "description": "## Overview\n\nPerspectiveGap evaluates whether a model can compose orchestration prompts for multi-agent systems while routing only the context each sub-agent needs.\n\n## Tasks\n\n- `perspective_gap_role_assignment`: select the visible fragment IDs for each role and return a JSON object.\n- `perspective_gap_prompt_writing`: write one markdown prompt section per role while including only the needed fragments.\n\n## Data\n\nThe benchmark uses the ModelScope dataset `evalscope/PerspectiveGap`, which contains the released `test` split. You can also pass `dataset_args..local_path` to a local JSONL mirror with the same fields.\n\n## Scoring\n\nScores are computed by `perspective_gap.scoring` from the official PerspectiveGap repository. The scorer is imported lazily so EvalScope can list benchmarks without installing the optional dependency.\n\nInstall the scorer before running evaluation: `pip install 'perspective-gap @ git+https://github.com/WhymustIhaveaname/PerspectiveGap.git@9c6921b3337ff3e6a6a453f68d117a8c1663135e'`. If you use uv, run `uv pip install 'perspective-gap @ git+https://github.com/WhymustIhaveaname/PerspectiveGap.git@9c6921b3337ff3e6a6a453f68d117a8c1663135e'`.", + "prompt_template": "{question}", + "system_prompt": "", + "few_shot_prompt_template": "", + "aggregation": "mean", + "extra_params": {}, + "sandbox_config": {}, + "category": "llm" + }, + "statistics": { + "total_samples": 220, + "subset_stats": [ + { + "name": "default", + "sample_count": 220, + "prompt_length_mean": 11398.63, + "prompt_length_min": 3909, + "prompt_length_max": 19086, + "prompt_length_std": 3729.43, + "target_length_mean": 172.75 + } + ], + "prompt_length": { + "mean": 11398.63, + "min": 3909, + "max": 19086, + "std": 3729.43 + }, + "target_length_mean": 172.75, + "computed_at": "2026-07-08T10:58:30.164760" + }, + "sample_example": { + "data": { + "input": [ + { + "id": "f0db45f1", + "content": "I need you to set up a 2-agent pipeline for bug bounty.\nThe two roles are coder and reviewer: coder is responsible for finding bugs, reviewer is responsible for judging whether what the coder found counts as a bug.\nAfter each bug the coder fi ... [TRUNCATED 3723 chars] ... of fragment content. Brief connective text between fragments (e.g., \"Then: ...\", \"Note: ...\") is fine. Format: one markdown section per role, with the role name as an h1 header (e.g. `# coder`). Output only the headered prompts, no preamble.\n" + } + ], + "target": "{\"coder\": [\"f5\", \"f6\", \"f7\", \"f1\"], \"reviewer\": [\"f5\", \"f3\", \"f4\"]}", + "id": 0, + "group_id": 0, + "metadata": { + "task": "prompt_writing", + "evaluation_id": "pg_000__seed_1", + "scenario_id": "pg_000", + "shuffle_seed": 1, + "roles": [ + "coder", + "reviewer" + ], + "fragments": [ + { + "id": "f1", + "text": "After finding a bug, submit it to a **fresh** reviewer for review. Do not review your own report.\n\n**After the review returns:** execute all action steps in the reviewer's output.", + "is_distractor": false + }, + { + "id": "f2", + "text": "Claude responds well to clear, explicit instructions. Being specific about your desired output can help enhance results. If you want \"above and beyond\" behavior, explicitly request it rather than relying on the model to infer this from vague ... [TRUNCATED 276 chars] ... w it. If they'd be confused, Claude will be too.\n\n- Be specific about the desired output format and constraints.\n- Provide instructions as sequential steps using numbered lists or bullet points when the order or completeness of steps matters.", + "is_distractor": true + }, + { + "id": "f3", + "text": "Audit the report you are given.", + "is_distractor": false + }, + { + "id": "f4", + "text": "Check the following dimensions one by one:\n\n| Dimension | Requirement |\n|-----------|-------------|\n| Scope | Within consensus-related code directories, or outside but directly affecting consensus logic |\n| Code references | Cited code snippe ... [TRUNCATED 1134 chars] ... ying victim code\n- Code references inconsistent with the latest testnet branch\n- PoC is incomplete and requires the reviewer to manually assemble it\n- Attack effect is equivalent to a Byzantine node simply going offline, with no amplification", + "is_distractor": false + }, + { + "id": "f5", + "text": "You are participating in a blockchain consensus-layer bug bounty contest.\n\nRead the contest rules first: ./contest-rules.md", + "is_distractor": false + }, + { + "id": "f6", + "text": "Find vulnerability in the code and write it up as a report.", + "is_distractor": false + }, + { + "id": "f7", + "text": "This project has been running stably for many years — any obvious fatal bug would have killed it long ago. So:\n\n- Don't report problems that are obvious at a glance (e.g. \"some map is never erased\") — if it were that obvious, the project team would have fixed it themselves\n- Truly valuable vulnerabilities hide in non-obvious interactions, races, and boundary conditions", + "is_distractor": false + } + ], + "distractor_id": "f2", + "reference_need_sets": { + "coder": [ + "f5", + "f6", + "f7", + "f1" + ], + "reviewer": [ + "f5", + "f3", + "f4" + ] + } + } + }, + "subset": "default", + "truncated": false + }, + "readme": { + "en": "# PerspectiveGap Prompt Writing\n\n## Overview\n\nPerspectiveGap evaluates whether a model can compose orchestration prompts for multi-agent systems while routing only the context each sub-agent needs.\n\n## Tasks\n\n- `perspective_gap_role_assignment`: select the visible fragment IDs for each role and return a JSON object.\n- `perspective_gap_prompt_writing`: write one markdown prompt section per role while including only the needed fragments.\n\n## Data\n\nThe benchmark uses the ModelScope dataset `evalscope/PerspectiveGap`, which contains the released `test` split. You can also pass `dataset_args..local_path` to a local JSONL mirror with the same fields.\n\n## Scoring\n\nScores are computed by `perspective_gap.scoring` from the official PerspectiveGap repository. The scorer is imported lazily so EvalScope can list benchmarks without installing the optional dependency.\n\nInstall the scorer before running evaluation: `pip install 'perspective-gap @ git+https://github.com/WhymustIhaveaname/PerspectiveGap.git@9c6921b3337ff3e6a6a453f68d117a8c1663135e'`. If you use uv, run `uv pip install 'perspective-gap @ git+https://github.com/WhymustIhaveaname/PerspectiveGap.git@9c6921b3337ff3e6a6a453f68d117a8c1663135e'`.\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `perspective_gap_prompt_writing` |\n| **Dataset ID** | [evalscope/PerspectiveGap](https://modelscope.cn/datasets/evalscope/PerspectiveGap/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2606.08878) |\n| **Tags** | `Agent`, `InstructionFollowing` |\n| **Metrics** | `strict_pass`, `net_match_score`, `required_coverage`, `boundary_precision`, `distractor_leakage` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 220 |\n| Prompt Length (Mean) | 11398.63 chars |\n| Prompt Length (Min/Max) | 3909 / 19086 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"f0db45f1\",\n \"content\": \"I need you to set up a 2-agent pipeline for bug bounty.\\nThe two roles are coder and reviewer: coder is responsible for finding bugs, reviewer is responsible for judging whether what the coder found counts as a bug.\\nAfter each bug the coder fi ... [TRUNCATED 3723 chars] ... of fragment content. Brief connective text between fragments (e.g., \\\"Then: ...\\\", \\\"Note: ...\\\") is fine. Format: one markdown section per role, with the role name as an h1 header (e.g. `# coder`). Output only the headered prompts, no preamble.\\n\"\n }\n ],\n \"target\": \"{\\\"coder\\\": [\\\"f5\\\", \\\"f6\\\", \\\"f7\\\", \\\"f1\\\"], \\\"reviewer\\\": [\\\"f5\\\", \\\"f3\\\", \\\"f4\\\"]}\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"task\": \"prompt_writing\",\n \"evaluation_id\": \"pg_000__seed_1\",\n \"scenario_id\": \"pg_000\",\n \"shuffle_seed\": 1,\n \"roles\": [\n \"coder\",\n \"reviewer\"\n ],\n \"fragments\": [\n {\n \"id\": \"f1\",\n \"text\": \"After finding a bug, submit it to a **fresh** reviewer for review. Do not review your own report.\\n\\n**After the review returns:** execute all action steps in the reviewer's output.\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f2\",\n \"text\": \"Claude responds well to clear, explicit instructions. Being specific about your desired output can help enhance results. If you want \\\"above and beyond\\\" behavior, explicitly request it rather than relying on the model to infer this from vague ... [TRUNCATED 276 chars] ... w it. If they'd be confused, Claude will be too.\\n\\n- Be specific about the desired output format and constraints.\\n- Provide instructions as sequential steps using numbered lists or bullet points when the order or completeness of steps matters.\",\n \"is_distractor\": true\n },\n {\n \"id\": \"f3\",\n \"text\": \"Audit the report you are given.\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f4\",\n \"text\": \"Check the following dimensions one by one:\\n\\n| Dimension | Requirement |\\n|-----------|-------------|\\n| Scope | Within consensus-related code directories, or outside but directly affecting consensus logic |\\n| Code references | Cited code snippe ... [TRUNCATED 1134 chars] ... ying victim code\\n- Code references inconsistent with the latest testnet branch\\n- PoC is incomplete and requires the reviewer to manually assemble it\\n- Attack effect is equivalent to a Byzantine node simply going offline, with no amplification\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f5\",\n \"text\": \"You are participating in a blockchain consensus-layer bug bounty contest.\\n\\nRead the contest rules first: ./contest-rules.md\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f6\",\n \"text\": \"Find vulnerability in the code and write it up as a report.\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f7\",\n \"text\": \"This project has been running stably for many years — any obvious fatal bug would have killed it long ago. So:\\n\\n- Don't report problems that are obvious at a glance (e.g. \\\"some map is never erased\\\") — if it were that obvious, the project team would have fixed it themselves\\n- Truly valuable vulnerabilities hide in non-obvious interactions, races, and boundary conditions\",\n \"is_distractor\": false\n }\n ],\n \"distractor_id\": \"f2\",\n \"reference_need_sets\": {\n \"coder\": [\n \"f5\",\n \"f6\",\n \"f7\",\n \"f1\"\n ],\n \"reviewer\": [\n \"f5\",\n \"f3\",\n \"f4\"\n ]\n }\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets perspective_gap_prompt_writing \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['perspective_gap_prompt_writing'],\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# PerspectiveGap Prompt Writing\n\n## 概述\n\nPerspectiveGap 用于评估模型是否能够在为多智能体系统编写编排提示（orchestration prompts）时，仅向每个子智能体提供其所需的上下文。\n\n## 任务\n\n- `perspective_gap_role_assignment`：为每个角色选择可见的片段 ID，并返回一个 JSON 对象。\n- `perspective_gap_prompt_writing`：为每个角色编写一个 Markdown 格式的提示段落，且仅包含该角色所需的片段。\n\n## 数据\n\n该基准测试使用 ModelScope 数据集 [`evalscope/PerspectiveGap`](https://modelscope.cn/datasets/evalscope/PerspectiveGap/summary)，其中包含已发布的 `test` 划分。你也可以通过 `dataset_args..local_path` 参数传入一个具有相同字段结构的本地 JSONL 文件镜像。\n\n## 评分\n\n评分由官方 PerspectiveGap 仓库中的 `perspective_gap.scoring` 模块计算得出。该评分器采用惰性导入方式，以便 EvalScope 在未安装可选依赖的情况下也能列出基准测试。\n\n在运行评估前，请先安装评分器：\n```bash\npip install 'perspective-gap @ git+https://github.com/WhymustIhaveaname/PerspectiveGap.git@9c6921b3337ff3e6a6a453f68d117a8c1663135e'\n```\n\n如果你使用 uv，则运行：\n```bash\nuv pip install 'perspective-gap @ git+https://github.com/WhymustIhaveaname/PerspectiveGap.git@9c6921b3337ff3e6a6a453f68d117a8c1663135e'\n```\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `perspective_gap_prompt_writing` |\n| **数据集ID** | [evalscope/PerspectiveGap](https://modelscope.cn/datasets/evalscope/PerspectiveGap/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2606.08878) |\n| **标签** | `Agent`, `InstructionFollowing` |\n| **指标** | `strict_pass`, `net_match_score`, `required_coverage`, `boundary_precision`, `distractor_leakage` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 220 |\n| 提示词长度（平均） | 11398.63 字符 |\n| 提示词长度（最小/最大） | 3909 / 19086 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"f0db45f1\",\n \"content\": \"I need you to set up a 2-agent pipeline for bug bounty.\\nThe two roles are coder and reviewer: coder is responsible for finding bugs, reviewer is responsible for judging whether what the coder found counts as a bug.\\nAfter each bug the coder fi ... [TRUNCATED 3723 chars] ... of fragment content. Brief connective text between fragments (e.g., \\\"Then: ...\\\", \\\"Note: ...\\\") is fine. Format: one markdown section per role, with the role name as an h1 header (e.g. `# coder`). Output only the headered prompts, no preamble.\\n\"\n }\n ],\n \"target\": \"{\\\"coder\\\": [\\\"f5\\\", \\\"f6\\\", \\\"f7\\\", \\\"f1\\\"], \\\"reviewer\\\": [\\\"f5\\\", \\\"f3\\\", \\\"f4\\\"]}\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"task\": \"prompt_writing\",\n \"evaluation_id\": \"pg_000__seed_1\",\n \"scenario_id\": \"pg_000\",\n \"shuffle_seed\": 1,\n \"roles\": [\n \"coder\",\n \"reviewer\"\n ],\n \"fragments\": [\n {\n \"id\": \"f1\",\n \"text\": \"After finding a bug, submit it to a **fresh** reviewer for review. Do not review your own report.\\n\\n**After the review returns:** execute all action steps in the reviewer's output.\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f2\",\n \"text\": \"Claude responds well to clear, explicit instructions. Being specific about your desired output can help enhance results. If you want \\\"above and beyond\\\" behavior, explicitly request it rather than relying on the model to infer this from vague ... [TRUNCATED 276 chars] ... w it. If they'd be confused, Claude will be too.\\n\\n- Be specific about the desired output format and constraints.\\n- Provide instructions as sequential steps using numbered lists or bullet points when the order or completeness of steps matters.\",\n \"is_distractor\": true\n },\n {\n \"id\": \"f3\",\n \"text\": \"Audit the report you are given.\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f4\",\n \"text\": \"Check the following dimensions one by one:\\n\\n| Dimension | Requirement |\\n|-----------|-------------|\\n| Scope | Within consensus-related code directories, or outside but directly affecting consensus logic |\\n| Code references | Cited code snippe ... [TRUNCATED 1134 chars] ... ying victim code\\n- Code references inconsistent with the latest testnet branch\\n- PoC is incomplete and requires the reviewer to manually assemble it\\n- Attack effect is equivalent to a Byzantine node simply going offline, with no amplification\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f5\",\n \"text\": \"You are participating in a blockchain consensus-layer bug bounty contest.\\n\\nRead the contest rules first: ./contest-rules.md\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f6\",\n \"text\": \"Find vulnerability in the code and write it up as a report.\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f7\",\n \"text\": \"This project has been running stably for many years — any obvious fatal bug would have killed it long ago. So:\\n\\n- Don't report problems that are obvious at a glance (e.g. \\\"some map is never erased\\\") — if it were that obvious, the project team would have fixed it themselves\\n- Truly valuable vulnerabilities hide in non-obvious interactions, races, and boundary conditions\",\n \"is_distractor\": false\n }\n ],\n \"distractor_id\": \"f2\",\n \"reference_need_sets\": {\n \"coder\": [\n \"f5\",\n \"f6\",\n \"f7\",\n \"f1\"\n ],\n \"reviewer\": [\n \"f5\",\n \"f3\",\n \"f4\"\n ]\n }\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets perspective_gap_prompt_writing \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['perspective_gap_prompt_writing'],\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "96d325627149904ba7edf3f3b9a772c5", + "needs_translation": false + }, + "updated_at": "2026-07-08T10:58:30.244193", + "translation_updated_at": "2026-07-08T10:58:35" +} diff --git a/evalscope/evalscope/benchmarks/_meta/perspective_gap_role_assignment.json b/evalscope/evalscope/benchmarks/_meta/perspective_gap_role_assignment.json new file mode 100644 index 0000000..e0753ba --- /dev/null +++ b/evalscope/evalscope/benchmarks/_meta/perspective_gap_role_assignment.json @@ -0,0 +1,138 @@ +{ + "meta": { + "pretty_name": "PerspectiveGap Role Assignment", + "dataset_id": "evalscope/PerspectiveGap", + "paper_url": "https://arxiv.org/abs/2606.08878", + "tags": [ + "Agent", + "InstructionFollowing" + ], + "metrics": [ + "strict_pass", + "net_match_score", + "required_coverage", + "boundary_precision", + "distractor_leakage" + ], + "few_shot_num": 0, + "eval_split": "test", + "train_split": "", + "subset_list": [ + "default" + ], + "description": "## Overview\n\nPerspectiveGap evaluates whether a model can compose orchestration prompts for multi-agent systems while routing only the context each sub-agent needs.\n\n## Tasks\n\n- `perspective_gap_role_assignment`: select the visible fragment IDs for each role and return a JSON object.\n- `perspective_gap_prompt_writing`: write one markdown prompt section per role while including only the needed fragments.\n\n## Data\n\nThe benchmark uses the ModelScope dataset `evalscope/PerspectiveGap`, which contains the released `test` split. You can also pass `dataset_args..local_path` to a local JSONL mirror with the same fields.\n\n## Scoring\n\nScores are computed by `perspective_gap.scoring` from the official PerspectiveGap repository. The scorer is imported lazily so EvalScope can list benchmarks without installing the optional dependency.\n\nInstall the scorer before running evaluation: `pip install 'perspective-gap @ git+https://github.com/WhymustIhaveaname/PerspectiveGap.git@9c6921b3337ff3e6a6a453f68d117a8c1663135e'`. If you use uv, run `uv pip install 'perspective-gap @ git+https://github.com/WhymustIhaveaname/PerspectiveGap.git@9c6921b3337ff3e6a6a453f68d117a8c1663135e'`.", + "prompt_template": "{question}", + "system_prompt": "", + "few_shot_prompt_template": "", + "aggregation": "mean", + "extra_params": {}, + "sandbox_config": {}, + "category": "llm" + }, + "statistics": { + "total_samples": 220, + "subset_stats": [ + { + "name": "default", + "sample_count": 220, + "prompt_length_mean": 11329.68, + "prompt_length_min": 3714, + "prompt_length_max": 19088, + "prompt_length_std": 3788.43, + "target_length_mean": 172.75 + } + ], + "prompt_length": { + "mean": 11329.68, + "min": 3714, + "max": 19088, + "std": 3788.43 + }, + "target_length_mean": 172.75, + "computed_at": "2026-07-08T10:58:30.168099" + }, + "sample_example": { + "data": { + "input": [ + { + "id": "0ce12f47", + "content": "I need you to set up a 2-agent pipeline for bug bounty.\nThe two roles are coder and reviewer: coder is responsible for finding bugs, reviewer is responsible for judging whether what the coder found counts as a bug.\nAfter each bug the coder fi ... [TRUNCATED 3528 chars] ... t's prompt should contain only the information that agent needs to do its job. For each sub-agent (coder, reviewer), which items should go in its prompt? Respond with a JSON object like {\"coder\": [\"f?\"], \"reviewer\": [\"f?\"]}. No other output.\n" + } + ], + "target": "{\"coder\": [\"f5\", \"f6\", \"f7\", \"f1\"], \"reviewer\": [\"f5\", \"f3\", \"f4\"]}", + "id": 0, + "group_id": 0, + "metadata": { + "task": "role_assignment", + "evaluation_id": "pg_000__seed_1", + "scenario_id": "pg_000", + "shuffle_seed": 1, + "roles": [ + "coder", + "reviewer" + ], + "fragments": [ + { + "id": "f1", + "text": "After finding a bug, submit it to a **fresh** reviewer for review. Do not review your own report.\n\n**After the review returns:** execute all action steps in the reviewer's output.", + "is_distractor": false + }, + { + "id": "f2", + "text": "Claude responds well to clear, explicit instructions. Being specific about your desired output can help enhance results. If you want \"above and beyond\" behavior, explicitly request it rather than relying on the model to infer this from vague ... [TRUNCATED 276 chars] ... w it. If they'd be confused, Claude will be too.\n\n- Be specific about the desired output format and constraints.\n- Provide instructions as sequential steps using numbered lists or bullet points when the order or completeness of steps matters.", + "is_distractor": true + }, + { + "id": "f3", + "text": "Audit the report you are given.", + "is_distractor": false + }, + { + "id": "f4", + "text": "Check the following dimensions one by one:\n\n| Dimension | Requirement |\n|-----------|-------------|\n| Scope | Within consensus-related code directories, or outside but directly affecting consensus logic |\n| Code references | Cited code snippe ... [TRUNCATED 1134 chars] ... ying victim code\n- Code references inconsistent with the latest testnet branch\n- PoC is incomplete and requires the reviewer to manually assemble it\n- Attack effect is equivalent to a Byzantine node simply going offline, with no amplification", + "is_distractor": false + }, + { + "id": "f5", + "text": "You are participating in a blockchain consensus-layer bug bounty contest.\n\nRead the contest rules first: ./contest-rules.md", + "is_distractor": false + }, + { + "id": "f6", + "text": "Find vulnerability in the code and write it up as a report.", + "is_distractor": false + }, + { + "id": "f7", + "text": "This project has been running stably for many years — any obvious fatal bug would have killed it long ago. So:\n\n- Don't report problems that are obvious at a glance (e.g. \"some map is never erased\") — if it were that obvious, the project team would have fixed it themselves\n- Truly valuable vulnerabilities hide in non-obvious interactions, races, and boundary conditions", + "is_distractor": false + } + ], + "distractor_id": "f2", + "reference_need_sets": { + "coder": [ + "f5", + "f6", + "f7", + "f1" + ], + "reviewer": [ + "f5", + "f3", + "f4" + ] + } + } + }, + "subset": "default", + "truncated": false + }, + "readme": { + "en": "# PerspectiveGap Role Assignment\n\n## Overview\n\nPerspectiveGap evaluates whether a model can compose orchestration prompts for multi-agent systems while routing only the context each sub-agent needs.\n\n## Tasks\n\n- `perspective_gap_role_assignment`: select the visible fragment IDs for each role and return a JSON object.\n- `perspective_gap_prompt_writing`: write one markdown prompt section per role while including only the needed fragments.\n\n## Data\n\nThe benchmark uses the ModelScope dataset `evalscope/PerspectiveGap`, which contains the released `test` split. You can also pass `dataset_args..local_path` to a local JSONL mirror with the same fields.\n\n## Scoring\n\nScores are computed by `perspective_gap.scoring` from the official PerspectiveGap repository. The scorer is imported lazily so EvalScope can list benchmarks without installing the optional dependency.\n\nInstall the scorer before running evaluation: `pip install 'perspective-gap @ git+https://github.com/WhymustIhaveaname/PerspectiveGap.git@9c6921b3337ff3e6a6a453f68d117a8c1663135e'`. If you use uv, run `uv pip install 'perspective-gap @ git+https://github.com/WhymustIhaveaname/PerspectiveGap.git@9c6921b3337ff3e6a6a453f68d117a8c1663135e'`.\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `perspective_gap_role_assignment` |\n| **Dataset ID** | [evalscope/PerspectiveGap](https://modelscope.cn/datasets/evalscope/PerspectiveGap/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2606.08878) |\n| **Tags** | `Agent`, `InstructionFollowing` |\n| **Metrics** | `strict_pass`, `net_match_score`, `required_coverage`, `boundary_precision`, `distractor_leakage` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 220 |\n| Prompt Length (Mean) | 11329.68 chars |\n| Prompt Length (Min/Max) | 3714 / 19088 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"0ce12f47\",\n \"content\": \"I need you to set up a 2-agent pipeline for bug bounty.\\nThe two roles are coder and reviewer: coder is responsible for finding bugs, reviewer is responsible for judging whether what the coder found counts as a bug.\\nAfter each bug the coder fi ... [TRUNCATED 3528 chars] ... t's prompt should contain only the information that agent needs to do its job. For each sub-agent (coder, reviewer), which items should go in its prompt? Respond with a JSON object like {\\\"coder\\\": [\\\"f?\\\"], \\\"reviewer\\\": [\\\"f?\\\"]}. No other output.\\n\"\n }\n ],\n \"target\": \"{\\\"coder\\\": [\\\"f5\\\", \\\"f6\\\", \\\"f7\\\", \\\"f1\\\"], \\\"reviewer\\\": [\\\"f5\\\", \\\"f3\\\", \\\"f4\\\"]}\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"task\": \"role_assignment\",\n \"evaluation_id\": \"pg_000__seed_1\",\n \"scenario_id\": \"pg_000\",\n \"shuffle_seed\": 1,\n \"roles\": [\n \"coder\",\n \"reviewer\"\n ],\n \"fragments\": [\n {\n \"id\": \"f1\",\n \"text\": \"After finding a bug, submit it to a **fresh** reviewer for review. Do not review your own report.\\n\\n**After the review returns:** execute all action steps in the reviewer's output.\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f2\",\n \"text\": \"Claude responds well to clear, explicit instructions. Being specific about your desired output can help enhance results. If you want \\\"above and beyond\\\" behavior, explicitly request it rather than relying on the model to infer this from vague ... [TRUNCATED 276 chars] ... w it. If they'd be confused, Claude will be too.\\n\\n- Be specific about the desired output format and constraints.\\n- Provide instructions as sequential steps using numbered lists or bullet points when the order or completeness of steps matters.\",\n \"is_distractor\": true\n },\n {\n \"id\": \"f3\",\n \"text\": \"Audit the report you are given.\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f4\",\n \"text\": \"Check the following dimensions one by one:\\n\\n| Dimension | Requirement |\\n|-----------|-------------|\\n| Scope | Within consensus-related code directories, or outside but directly affecting consensus logic |\\n| Code references | Cited code snippe ... [TRUNCATED 1134 chars] ... ying victim code\\n- Code references inconsistent with the latest testnet branch\\n- PoC is incomplete and requires the reviewer to manually assemble it\\n- Attack effect is equivalent to a Byzantine node simply going offline, with no amplification\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f5\",\n \"text\": \"You are participating in a blockchain consensus-layer bug bounty contest.\\n\\nRead the contest rules first: ./contest-rules.md\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f6\",\n \"text\": \"Find vulnerability in the code and write it up as a report.\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f7\",\n \"text\": \"This project has been running stably for many years — any obvious fatal bug would have killed it long ago. So:\\n\\n- Don't report problems that are obvious at a glance (e.g. \\\"some map is never erased\\\") — if it were that obvious, the project team would have fixed it themselves\\n- Truly valuable vulnerabilities hide in non-obvious interactions, races, and boundary conditions\",\n \"is_distractor\": false\n }\n ],\n \"distractor_id\": \"f2\",\n \"reference_need_sets\": {\n \"coder\": [\n \"f5\",\n \"f6\",\n \"f7\",\n \"f1\"\n ],\n \"reviewer\": [\n \"f5\",\n \"f3\",\n \"f4\"\n ]\n }\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets perspective_gap_role_assignment \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['perspective_gap_role_assignment'],\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# PerspectiveGap Role Assignment\n\n## 概述\n\nPerspectiveGap 用于评估模型是否能够在为多智能体系统编写编排提示时，仅向每个子智能体提供其所需上下文。\n\n## 任务\n\n- `perspective_gap_role_assignment`：为每个角色选择可见的片段 ID，并返回一个 JSON 对象。\n- `perspective_gap_prompt_writing`：为每个角色编写一段 Markdown 格式的提示，且仅包含所需的片段。\n\n## 数据\n\n该基准测试使用 ModelScope 数据集 [`evalscope/PerspectiveGap`](https://modelscope.cn/datasets/evalscope/PerspectiveGap/summary)，其中包含已发布的 `test` 划分。你也可以通过 `dataset_args..local_path` 参数传入具有相同字段结构的本地 JSONL 镜像文件。\n\n## 评分\n\n评分由官方 PerspectiveGap 仓库中的 `perspective_gap.scoring` 模块计算得出。该评分器采用懒加载方式导入，以便 EvalScope 在未安装可选依赖的情况下也能列出基准测试。\n\n在运行评估前，请先安装评分器：\n```bash\npip install 'perspective-gap @ git+https://github.com/WhymustIhaveaname/PerspectiveGap.git@9c6921b3337ff3e6a6a453f68d117a8c1663135e'\n```\n\n如果你使用 uv，则运行：\n```bash\nuv pip install 'perspective-gap @ git+https://github.com/WhymustIhaveaname/PerspectiveGap.git@9c6921b3337ff3e6a6a453f68d117a8c1663135e'\n```\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `perspective_gap_role_assignment` |\n| **数据集ID** | [evalscope/PerspectiveGap](https://modelscope.cn/datasets/evalscope/PerspectiveGap/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2606.08878) |\n| **标签** | `Agent`, `InstructionFollowing` |\n| **指标** | `strict_pass`, `net_match_score`, `required_coverage`, `boundary_precision`, `distractor_leakage` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 220 |\n| 提示词长度（平均） | 11329.68 字符 |\n| 提示词长度（最小/最大） | 3714 / 19088 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"0ce12f47\",\n \"content\": \"I need you to set up a 2-agent pipeline for bug bounty.\\nThe two roles are coder and reviewer: coder is responsible for finding bugs, reviewer is responsible for judging whether what the coder found counts as a bug.\\nAfter each bug the coder fi ... [TRUNCATED 3528 chars] ... t's prompt should contain only the information that agent needs to do its job. For each sub-agent (coder, reviewer), which items should go in its prompt? Respond with a JSON object like {\\\"coder\\\": [\\\"f?\\\"], \\\"reviewer\\\": [\\\"f?\\\"]}. No other output.\\n\"\n }\n ],\n \"target\": \"{\\\"coder\\\": [\\\"f5\\\", \\\"f6\\\", \\\"f7\\\", \\\"f1\\\"], \\\"reviewer\\\": [\\\"f5\\\", \\\"f3\\\", \\\"f4\\\"]}\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"task\": \"role_assignment\",\n \"evaluation_id\": \"pg_000__seed_1\",\n \"scenario_id\": \"pg_000\",\n \"shuffle_seed\": 1,\n \"roles\": [\n \"coder\",\n \"reviewer\"\n ],\n \"fragments\": [\n {\n \"id\": \"f1\",\n \"text\": \"After finding a bug, submit it to a **fresh** reviewer for review. Do not review your own report.\\n\\n**After the review returns:** execute all action steps in the reviewer's output.\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f2\",\n \"text\": \"Claude responds well to clear, explicit instructions. Being specific about your desired output can help enhance results. If you want \\\"above and beyond\\\" behavior, explicitly request it rather than relying on the model to infer this from vague ... [TRUNCATED 276 chars] ... w it. If they'd be confused, Claude will be too.\\n\\n- Be specific about the desired output format and constraints.\\n- Provide instructions as sequential steps using numbered lists or bullet points when the order or completeness of steps matters.\",\n \"is_distractor\": true\n },\n {\n \"id\": \"f3\",\n \"text\": \"Audit the report you are given.\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f4\",\n \"text\": \"Check the following dimensions one by one:\\n\\n| Dimension | Requirement |\\n|-----------|-------------|\\n| Scope | Within consensus-related code directories, or outside but directly affecting consensus logic |\\n| Code references | Cited code snippe ... [TRUNCATED 1134 chars] ... ying victim code\\n- Code references inconsistent with the latest testnet branch\\n- PoC is incomplete and requires the reviewer to manually assemble it\\n- Attack effect is equivalent to a Byzantine node simply going offline, with no amplification\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f5\",\n \"text\": \"You are participating in a blockchain consensus-layer bug bounty contest.\\n\\nRead the contest rules first: ./contest-rules.md\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f6\",\n \"text\": \"Find vulnerability in the code and write it up as a report.\",\n \"is_distractor\": false\n },\n {\n \"id\": \"f7\",\n \"text\": \"This project has been running stably for many years — any obvious fatal bug would have killed it long ago. So:\\n\\n- Don't report problems that are obvious at a glance (e.g. \\\"some map is never erased\\\") — if it were that obvious, the project team would have fixed it themselves\\n- Truly valuable vulnerabilities hide in non-obvious interactions, races, and boundary conditions\",\n \"is_distractor\": false\n }\n ],\n \"distractor_id\": \"f2\",\n \"reference_need_sets\": {\n \"coder\": [\n \"f5\",\n \"f6\",\n \"f7\",\n \"f1\"\n ],\n \"reviewer\": [\n \"f5\",\n \"f3\",\n \"f4\"\n ]\n }\n }\n}\n```\n\n## 提示模板\n\n**提示模板:**\n```text\n{question}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets perspective_gap_role_assignment \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['perspective_gap_role_assignment'],\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "2151e29a336c1fb9ee066620cebb03cd", + "needs_translation": false + }, + "updated_at": "2026-07-08T10:58:30.242531", + "translation_updated_at": "2026-07-08T10:58:35" +} diff --git a/evalscope/evalscope/benchmarks/_meta/piqa.json b/evalscope/evalscope/benchmarks/_meta/piqa.json index 185da31..553b8d3 100644 --- a/evalscope/evalscope/benchmarks/_meta/piqa.json +++ b/evalscope/evalscope/benchmarks/_meta/piqa.json @@ -76,4 +76,4 @@ }, "updated_at": "2026-01-28T17:31:32.409396", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/poly_math.json b/evalscope/evalscope/benchmarks/_meta/poly_math.json index 5b1f371..c428947 100644 --- a/evalscope/evalscope/benchmarks/_meta/poly_math.json +++ b/evalscope/evalscope/benchmarks/_meta/poly_math.json @@ -736,4 +736,4 @@ }, "updated_at": "2026-01-28T17:31:32.410382", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/pope.json b/evalscope/evalscope/benchmarks/_meta/pope.json index 1e17a9d..b715d21 100644 --- a/evalscope/evalscope/benchmarks/_meta/pope.json +++ b/evalscope/evalscope/benchmarks/_meta/pope.json @@ -236,4 +236,4 @@ }, "updated_at": "2026-01-28T17:31:32.412181", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/process_bench.json b/evalscope/evalscope/benchmarks/_meta/process_bench.json index f5e9b9f..c93a11e 100644 --- a/evalscope/evalscope/benchmarks/_meta/process_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/process_bench.json @@ -112,4 +112,4 @@ }, "updated_at": "2026-01-28T17:31:32.431671", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/pubmedqa.json b/evalscope/evalscope/benchmarks/_meta/pubmedqa.json index bacb2a7..2869e5a 100644 --- a/evalscope/evalscope/benchmarks/_meta/pubmedqa.json +++ b/evalscope/evalscope/benchmarks/_meta/pubmedqa.json @@ -83,4 +83,4 @@ }, "updated_at": "2026-01-28T17:31:32.433353", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/qasc.json b/evalscope/evalscope/benchmarks/_meta/qasc.json index 676e242..061da99 100644 --- a/evalscope/evalscope/benchmarks/_meta/qasc.json +++ b/evalscope/evalscope/benchmarks/_meta/qasc.json @@ -81,4 +81,4 @@ }, "updated_at": "2026-01-28T17:31:32.434105", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/race.json b/evalscope/evalscope/benchmarks/_meta/race.json index 70b81b3..81b3f5c 100644 --- a/evalscope/evalscope/benchmarks/_meta/race.json +++ b/evalscope/evalscope/benchmarks/_meta/race.json @@ -89,4 +89,4 @@ }, "updated_at": "2026-01-28T17:31:32.435575", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/real_world_qa.json b/evalscope/evalscope/benchmarks/_meta/real_world_qa.json index 8a3209d..f6c37f3 100644 --- a/evalscope/evalscope/benchmarks/_meta/real_world_qa.json +++ b/evalscope/evalscope/benchmarks/_meta/real_world_qa.json @@ -145,4 +145,4 @@ }, "updated_at": "2026-01-28T17:31:32.438867", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/refcoco.json b/evalscope/evalscope/benchmarks/_meta/refcoco.json index e6481bb..9fdb711 100644 --- a/evalscope/evalscope/benchmarks/_meta/refcoco.json +++ b/evalscope/evalscope/benchmarks/_meta/refcoco.json @@ -311,4 +311,4 @@ }, "updated_at": "2026-01-28T17:31:32.447199", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/researchrubrics.json b/evalscope/evalscope/benchmarks/_meta/researchrubrics.json new file mode 100644 index 0000000..a80d9d4 --- /dev/null +++ b/evalscope/evalscope/benchmarks/_meta/researchrubrics.json @@ -0,0 +1,124 @@ +{ + "meta": { + "pretty_name": "ResearchRubrics", + "dataset_id": "evalscope/researchrubrics", + "paper_url": "https://arxiv.org/abs/2511.07685", + "tags": [ + "Agent", + "MultiTurn", + "Retrieval", + "Reasoning" + ], + "metrics": [ + "compliance_score" + ], + "few_shot_num": 0, + "eval_split": "train", + "train_split": "", + "subset_list": [ + "default" + ], + "description": "\n## Overview\n\nResearchRubrics evaluates Deep Research agents on realistic, open-ended research tasks. Each task pairs a user prompt\nwith expert-written, fine-grained rubrics covering explicit and implicit requirements, information synthesis,\nreferences, communication quality, and instruction following.\n\n## Task Description\n\n- **Task Type**: Multi-turn research agent / long-form report generation\n- **Input**: One open-ended research prompt\n- **Output**: A Markdown research report produced after iterative tool use\n- **Dataset**: 101 tasks and 2,593 weighted rubric criteria\n- **Metric**: Binary rubric compliance score\n\n## Agent Runtime\n\n- Uses EvalScope's built-in agent runtime by default and does not require ``agent_config``. The agent can use ``bash``\n to access the network, gather information, and produce a final report.\n- The default runtime uses the host network and a temporary working directory, but does not provide complete filesystem\n isolation. Do not run untrusted models on shared or sensitive machines.\n- The default strategy is ``function_calling`` with a 50-step limit. Use ``NativeAgentConfig`` to override the strategy\n or step limit; ``react`` is also available. Both strategies require native function calling support.\n- Add dedicated search or web-fetching tools through ``NativeAgentConfig``, or use ``ExternalAgentConfig`` to run the\n task with another agent framework.\n- When the step limit is reached, the model is asked to produce a final report from the information already collected so\n the result can still be reviewed and scored.\n\n## Evaluation Notes\n\n- ResearchRubrics requires ``judge_model_args`` and ``judge_strategy='auto'`` or ``'llm'``. Gemini 2.5 Pro is the\n recommended judge for comparison with the paper, but no provider or model is hard-coded.\n- Every rubric is graded independently as Satisfied (1) or Not Satisfied (0), matching the public binary grader. The\n paper's ternary scores are not directly comparable.\n- Negative-weight criteria subtract from the numerator when the undesirable behavior is present. Scores are not\n clipped.\n- Long reports are evaluated with the official chunk-evidence-synthesis approach when they exceed the configured judge\n context threshold.\n- A full run performs 2,593 rubric evaluations and can be expensive. Current-events tasks are also sensitive to the\n date and web sources available at evaluation time.\n\n## Configuration\n\n- ``judge_context_limit``: 150,000 estimated tokens\n- ``judge_chunk_size``: 100,000 estimated tokens\n- ``judge_retries``: 3 attempts per judge request\n\nThe judge must be configured explicitly. For example:\n\n```python\nfrom evalscope import TaskConfig, run_task\n\nrun_task(TaskConfig(\n model='YOUR_AGENT_MODEL',\n datasets=['researchrubrics'],\n judge_strategy='llm',\n judge_model_args={\n 'model_id': 'YOUR_JUDGE_MODEL',\n 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL',\n 'api_key': 'YOUR_JUDGE_API_KEY',\n 'generation_config': {'temperature': 0.0},\n },\n limit=1,\n))\n```\n\nResources: [Paper](https://arxiv.org/abs/2511.07685) |\n[GitHub](https://github.com/scaleapi/researchrubrics) |\n[Dataset](https://modelscope.cn/datasets/evalscope/researchrubrics)\n", + "prompt_template": "{question}", + "system_prompt": "", + "few_shot_prompt_template": "", + "aggregation": "mean", + "extra_params": { + "judge_context_limit": { + "type": "int", + "description": "Estimated token limit before rubric judging switches to chunking.", + "value": 150000 + }, + "judge_chunk_size": { + "type": "int", + "description": "Maximum estimated tokens in each document chunk sent to the judge.", + "value": 100000 + }, + "judge_retries": { + "type": "int", + "description": "Maximum attempts for each rubric judge request and JSON parse.", + "value": 3 + } + }, + "sandbox_config": {}, + "agent_config": { + "strategy": "function_calling", + "max_steps": 50 + }, + "category": "agent" + }, + "statistics": { + "total_samples": 101, + "subset_stats": [ + { + "name": "default", + "sample_count": 101, + "prompt_length_mean": 555.35, + "prompt_length_min": 102, + "prompt_length_max": 1747, + "prompt_length_std": 365.97, + "target_length_mean": 7807.24 + } + ], + "prompt_length": { + "mean": 555.35, + "min": 102, + "max": 1747, + "std": 365.97 + }, + "target_length_mean": 7807.24, + "computed_at": "2026-07-10T16:17:28.326792" + }, + "sample_example": { + "data": { + "input": [ + { + "id": "f1132c85", + "content": "I want to create a plan for July 4, 2025, i.e., Independence Day in Washington DC. I would like an itinerary of all the things to do and all the activities that are planned for Independence Day. Create a plan for the whole day and also extend it to the weekend, if required. Provide some reviews or explain why one should visit the place or engage in the activity. Add any additional information that is required." + } + ], + "target": "[{\"criterion\": \"The response covers the period from 9:00 AM or earlier through at least 10:00 PM on 4 July 2025\", \"weight\": 5.0, \"axis\": \"Explicit Criteria\"}, {\"criterion\": \"The response contains clear section headers for parts of the schedul ... [TRUNCATED 4470 chars] ... events from years other than 2025 (e.g., seeing a miltiary parade, information about \\\"A Capital Fourth\\\" for 2024, information the parade route for 2023, referencing a cancellation from 2020).\", \"weight\": -4.0, \"axis\": \"Explicit Criteria\"}]", + "id": 0, + "group_id": 0, + "tools": [ + { + "name": "bash", + "description": "Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.", + "parameters": { + "properties": { + "command": { + "type": "string", + "description": "The bash command to execute." + }, + "timeout": { + "type": "number", + "description": "Maximum execution time in seconds (default: 60).", + "default": 60 + } + }, + "required": [ + "command" + ] + } + } + ], + "metadata": { + "sample_id": "6847465956a0f6376a605427", + "domain": "Current Events", + "conceptual_breadth": "Simple", + "logical_nesting": "Intermediate", + "exploration": "Medium" + } + }, + "subset": "default", + "truncated": true + }, + "readme": { + "en": "# ResearchRubrics\n\n\n## Overview\n\nResearchRubrics evaluates Deep Research agents on realistic, open-ended research tasks. Each task pairs a user prompt\nwith expert-written, fine-grained rubrics covering explicit and implicit requirements, information synthesis,\nreferences, communication quality, and instruction following.\n\n## Task Description\n\n- **Task Type**: Multi-turn research agent / long-form report generation\n- **Input**: One open-ended research prompt\n- **Output**: A Markdown research report produced after iterative tool use\n- **Dataset**: 101 tasks and 2,593 weighted rubric criteria\n- **Metric**: Binary rubric compliance score\n\n## Agent Runtime\n\n- Uses EvalScope's built-in agent runtime by default and does not require ``agent_config``. The agent can use ``bash``\n to access the network, gather information, and produce a final report.\n- The default runtime uses the host network and a temporary working directory, but does not provide complete filesystem\n isolation. Do not run untrusted models on shared or sensitive machines.\n- The default strategy is ``function_calling`` with a 50-step limit. Use ``NativeAgentConfig`` to override the strategy\n or step limit; ``react`` is also available. Both strategies require native function calling support.\n- Add dedicated search or web-fetching tools through ``NativeAgentConfig``, or use ``ExternalAgentConfig`` to run the\n task with another agent framework.\n- When the step limit is reached, the model is asked to produce a final report from the information already collected so\n the result can still be reviewed and scored.\n\n## Evaluation Notes\n\n- ResearchRubrics requires ``judge_model_args`` and ``judge_strategy='auto'`` or ``'llm'``. Gemini 2.5 Pro is the\n recommended judge for comparison with the paper, but no provider or model is hard-coded.\n- Every rubric is graded independently as Satisfied (1) or Not Satisfied (0), matching the public binary grader. The\n paper's ternary scores are not directly comparable.\n- Negative-weight criteria subtract from the numerator when the undesirable behavior is present. Scores are not\n clipped.\n- Long reports are evaluated with the official chunk-evidence-synthesis approach when they exceed the configured judge\n context threshold.\n- A full run performs 2,593 rubric evaluations and can be expensive. Current-events tasks are also sensitive to the\n date and web sources available at evaluation time.\n\n## Configuration\n\n- ``judge_context_limit``: 150,000 estimated tokens\n- ``judge_chunk_size``: 100,000 estimated tokens\n- ``judge_retries``: 3 attempts per judge request\n\nThe judge must be configured explicitly. For example:\n\n```python\nfrom evalscope import TaskConfig, run_task\n\nrun_task(TaskConfig(\n model='YOUR_AGENT_MODEL',\n datasets=['researchrubrics'],\n judge_strategy='llm',\n judge_model_args={\n 'model_id': 'YOUR_JUDGE_MODEL',\n 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL',\n 'api_key': 'YOUR_JUDGE_API_KEY',\n 'generation_config': {'temperature': 0.0},\n },\n limit=1,\n))\n```\n\nResources: [Paper](https://arxiv.org/abs/2511.07685) |\n[GitHub](https://github.com/scaleapi/researchrubrics) |\n[Dataset](https://modelscope.cn/datasets/evalscope/researchrubrics)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `researchrubrics` |\n| **Dataset ID** | [evalscope/researchrubrics](https://modelscope.cn/datasets/evalscope/researchrubrics/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2511.07685) |\n| **Tags** | `Agent`, `MultiTurn`, `Reasoning`, `Retrieval` |\n| **Metrics** | `compliance_score` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 101 |\n| Prompt Length (Mean) | 555.35 chars |\n| Prompt Length (Min/Max) | 102 / 1747 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"f1132c85\",\n \"content\": \"I want to create a plan for July 4, 2025, i.e., Independence Day in Washington DC. I would like an itinerary of all the things to do and all the activities that are planned for Independence Day. Create a plan for the whole day and also extend it to the weekend, if required. Provide some reviews or explain why one should visit the place or engage in the activity. Add any additional information that is required.\"\n }\n ],\n \"target\": \"[{\\\"criterion\\\": \\\"The response covers the period from 9:00 AM or earlier through at least 10:00 PM on 4 July 2025\\\", \\\"weight\\\": 5.0, \\\"axis\\\": \\\"Explicit Criteria\\\"}, {\\\"criterion\\\": \\\"The response contains clear section headers for parts of the schedul ... [TRUNCATED 4470 chars] ... events from years other than 2025 (e.g., seeing a miltiary parade, information about \\\\\\\"A Capital Fourth\\\\\\\" for 2024, information the parade route for 2023, referencing a cancellation from 2020).\\\", \\\"weight\\\": -4.0, \\\"axis\\\": \\\"Explicit Criteria\\\"}]\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"sample_id\": \"6847465956a0f6376a605427\",\n \"domain\": \"Current Events\",\n \"conceptual_breadth\": \"Simple\",\n \"logical_nesting\": \"Intermediate\",\n \"exploration\": \"Medium\"\n }\n}\n```\n\n*Note: Some content was truncated for display.*\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `judge_context_limit` | `int` | `150000` | Estimated token limit before rubric judging switches to chunking. |\n| `judge_chunk_size` | `int` | `100000` | Maximum estimated tokens in each document chunk sent to the judge. |\n| `judge_retries` | `int` | `3` | Maximum attempts for each rubric judge request and JSON parse. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets researchrubrics \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":50}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['researchrubrics'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=50,\n ),\n dataset_args={\n 'researchrubrics': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# ResearchRubrics\n\n\n## 概述\n\nResearchRubrics 用于评估深度研究（Deep Research）智能体在真实、开放式研究任务上的表现。每个任务包含一个用户提示，以及由专家编写的细粒度评分标准（rubrics），涵盖显式与隐式需求、信息整合、参考文献、沟通质量以及指令遵循等方面。\n\n## 任务描述\n\n- **任务类型**：多轮研究智能体 / 长篇报告生成 \n- **输入**：一个开放式研究提示 \n- **输出**：通过迭代使用工具生成的 Markdown 格式研究报告 \n- **数据集**：101 个任务，共包含 2,593 条带权重的评分标准 \n- **评估指标**：二元评分标准符合度得分（Binary rubric compliance score）\n\n## 智能体运行环境\n\n- 默认使用 EvalScope 内置的智能体运行时，无需提供 ``agent_config``。智能体可通过 ``bash`` 访问网络、收集信息并生成最终报告。\n- 默认运行时使用主机网络和临时工作目录，但不提供完整的文件系统隔离。请勿在共享或敏感机器上运行不可信模型。\n- 默认策略为 ``function_calling``，最多执行 50 步。可通过 ``NativeAgentConfig`` 覆盖策略或步数限制；也可选择 ``react`` 策略。两种策略均需模型原生支持函数调用。\n- 可通过 ``NativeAgentConfig`` 添加专用搜索或网页抓取工具，或使用 ``ExternalAgentConfig`` 在其他智能体框架中运行任务。\n- 当达到步数限制时，模型会被要求基于已收集的信息生成最终报告，以便后续评审和打分。\n\n## 评估说明\n\n- ResearchRubrics 需要显式配置 ``judge_model_args``，且 ``judge_strategy`` 必须设为 ``'auto'`` 或 ``'llm'``。论文推荐使用 Gemini 2.5 Pro 作为评判模型，但未硬编码任何特定提供商或模型。\n- 每条评分标准独立判为“满足”（1）或“不满足”（0），与公开的二元评分器一致。论文中使用的三元评分无法直接比较。\n- 负权重标准在出现不良行为时会从分子中扣除相应分数，且最终得分不会被截断。\n- 当报告长度超过配置的评判模型上下文阈值时，将采用官方的分块-证据-合成（chunk-evidence-synthesis）方法进行评估。\n- 完整运行需执行 2,593 次评分标准评估，成本较高。此外，涉及当前事件的任务对评估时的日期和可用网络资源较为敏感。\n\n## 配置\n\n- ``judge_context_limit``: 150,000 个估算 token \n- ``judge_chunk_size``: 100,000 个估算 token \n- ``judge_retries``: 每次评判请求最多重试 3 次 \n\n评判模型必须显式配置。例如：\n\n```python\nfrom evalscope import TaskConfig, run_task\n\nrun_task(TaskConfig(\n model='YOUR_AGENT_MODEL',\n datasets=['researchrubrics'],\n judge_strategy='llm',\n judge_model_args={\n 'model_id': 'YOUR_JUDGE_MODEL',\n 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL',\n 'api_key': 'YOUR_JUDGE_API_KEY',\n 'generation_config': {'temperature': 0.0},\n },\n limit=1,\n))\n```\n\n资源链接：[论文](https://arxiv.org/abs/2511.07685) |\n[GitHub](https://github.com/scaleapi/researchrubrics) |\n[数据集](https://modelscope.cn/datasets/evalscope/researchrubrics)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `researchrubrics` |\n| **数据集ID** | [evalscope/researchrubrics](https://modelscope.cn/datasets/evalscope/researchrubrics/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2511.07685) |\n| **标签** | `Agent`, `MultiTurn`, `Reasoning`, `Retrieval` |\n| **指标** | `compliance_score` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `train` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 101 |\n| 提示词长度（平均） | 555.35 字符 |\n| 提示词长度（最小/最大） | 102 / 1747 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"f1132c85\",\n \"content\": \"I want to create a plan for July 4, 2025, i.e., Independence Day in Washington DC. I would like an itinerary of all the things to do and all the activities that are planned for Independence Day. Create a plan for the whole day and also extend it to the weekend, if required. Provide some reviews or explain why one should visit the place or engage in the activity. Add any additional information that is required.\"\n }\n ],\n \"target\": \"[{\\\"criterion\\\": \\\"The response covers the period from 9:00 AM or earlier through at least 10:00 PM on 4 July 2025\\\", \\\"weight\\\": 5.0, \\\"axis\\\": \\\"Explicit Criteria\\\"}, {\\\"criterion\\\": \\\"The response contains clear section headers for parts of the schedul ... [TRUNCATED 4470 chars] ... events from years other than 2025 (e.g., seeing a miltiary parade, information about \\\\\\\"A Capital Fourth\\\\\\\" for 2024, information the parade route for 2023, referencing a cancellation from 2020).\\\", \\\"weight\\\": -4.0, \\\"axis\\\": \\\"Explicit Criteria\\\"}]\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"sample_id\": \"6847465956a0f6376a605427\",\n \"domain\": \"Current Events\",\n \"conceptual_breadth\": \"Simple\",\n \"logical_nesting\": \"Intermediate\",\n \"exploration\": \"Medium\"\n }\n}\n```\n\n*注：部分内容因展示需要已被截断。*\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `judge_context_limit` | `int` | `150000` | 评判前允许的最大估算 token 数，超过后启用分块处理。 |\n| `judge_chunk_size` | `int` | `100000` | 发送给评判模型的每个文档分块的最大估算 token 数。 |\n| `judge_retries` | `int` | `3` | 每次评分标准评判请求及 JSON 解析的最大重试次数。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets researchrubrics \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":50}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['researchrubrics'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=50,\n ),\n dataset_args={\n 'researchrubrics': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "e9d0de4af8581f3c694b43f1a3793259", + "needs_translation": false + }, + "updated_at": "2026-07-10T20:24:34.912361", + "translation_updated_at": "2026-07-10T20:24:42" +} diff --git a/evalscope/evalscope/benchmarks/_meta/scicode.json b/evalscope/evalscope/benchmarks/_meta/scicode.json index bdf8ff2..67e5c00 100644 --- a/evalscope/evalscope/benchmarks/_meta/scicode.json +++ b/evalscope/evalscope/benchmarks/_meta/scicode.json @@ -230,4 +230,4 @@ }, "updated_at": "2026-05-28T18:11:41.128926", "translation_updated_at": "2026-05-28T18:11:44" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/science_qa.json b/evalscope/evalscope/benchmarks/_meta/science_qa.json index 66ae81a..878a279 100644 --- a/evalscope/evalscope/benchmarks/_meta/science_qa.json +++ b/evalscope/evalscope/benchmarks/_meta/science_qa.json @@ -154,4 +154,4 @@ }, "updated_at": "2026-01-28T17:31:32.446817", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/sciq.json b/evalscope/evalscope/benchmarks/_meta/sciq.json index 036916b..9282e9b 100644 --- a/evalscope/evalscope/benchmarks/_meta/sciq.json +++ b/evalscope/evalscope/benchmarks/_meta/sciq.json @@ -78,4 +78,4 @@ }, "updated_at": "2026-01-28T17:31:32.447367", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/seed_bench_2_plus.json b/evalscope/evalscope/benchmarks/_meta/seed_bench_2_plus.json index aac2983..193134d 100644 --- a/evalscope/evalscope/benchmarks/_meta/seed_bench_2_plus.json +++ b/evalscope/evalscope/benchmarks/_meta/seed_bench_2_plus.json @@ -208,4 +208,4 @@ }, "updated_at": "2026-01-28T17:31:32.447929", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/seed_tts_eval.json b/evalscope/evalscope/benchmarks/_meta/seed_tts_eval.json index 41ddc6d..66a555b 100644 --- a/evalscope/evalscope/benchmarks/_meta/seed_tts_eval.json +++ b/evalscope/evalscope/benchmarks/_meta/seed_tts_eval.json @@ -152,4 +152,4 @@ }, "updated_at": "2026-06-01T14:19:12.711468", "translation_updated_at": "2026-06-01T14:19:29" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/simple_qa.json b/evalscope/evalscope/benchmarks/_meta/simple_qa.json index 38bcf72..c1fb6a3 100644 --- a/evalscope/evalscope/benchmarks/_meta/simple_qa.json +++ b/evalscope/evalscope/benchmarks/_meta/simple_qa.json @@ -82,4 +82,4 @@ }, "updated_at": "2026-01-28T17:31:32.457485", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/simple_vqa.json b/evalscope/evalscope/benchmarks/_meta/simple_vqa.json index 5f8287b..e857432 100644 --- a/evalscope/evalscope/benchmarks/_meta/simple_vqa.json +++ b/evalscope/evalscope/benchmarks/_meta/simple_vqa.json @@ -153,4 +153,4 @@ }, "updated_at": "2026-01-28T17:31:32.459286", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/siqa.json b/evalscope/evalscope/benchmarks/_meta/siqa.json index a16e99b..c57d556 100644 --- a/evalscope/evalscope/benchmarks/_meta/siqa.json +++ b/evalscope/evalscope/benchmarks/_meta/siqa.json @@ -77,4 +77,4 @@ }, "updated_at": "2026-01-28T17:31:32.461374", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/skillsbench.json b/evalscope/evalscope/benchmarks/_meta/skillsbench.json new file mode 100644 index 0000000..b6f1d8e --- /dev/null +++ b/evalscope/evalscope/benchmarks/_meta/skillsbench.json @@ -0,0 +1,93 @@ +{ + "meta": { + "pretty_name": "SkillsBench", + "dataset_id": "skillsbench", + "paper_url": null, + "tags": [ + "Agent", + "MultiTurn" + ], + "metrics": [ + "score" + ], + "few_shot_num": 0, + "eval_split": null, + "train_split": "", + "subset_list": [ + "default" + ], + "description": "\n## Overview\n\nSkillsBench evaluates whether coding agents can discover and apply task-bundled Agent Skills. Each task contains an\ninstruction, an optional skill directory, a Docker environment, an oracle solution, and a verifier. EvalScope builds the\ntask Docker image, runs the selected agent or oracle in that image, then executes the task verifier.\n\n## Task Description\n\n- **Task Type**: Agent skill usage / tool-assisted task completion\n- **Input**: The natural-language task prompt from `task.md`\n- **Output**: Files or state changes produced inside the task container, scored by the task verifier\n- **Dataset**: Local SkillsBench task repository supplied through `extra_params.tasks_dir`\n- **Environment**: Per-task Docker image built from `environment/Dockerfile`\n- **Skills**: Optional task-bundled skills from `environment/skills`\n- **Metric**: `score` from `/logs/verifier/reward.txt`; `success` is 1 when `score > 0`\n\n## Key Features\n\n- Builds or reuses a content-hashed Docker image for each selected task.\n- Runs the task in `no-skill` or `with-skill` mode without mixing the two conditions in one EvalScope run.\n- Injects task-bundled skills through EvalScope's agent skill runtime instead of baking runner-specific skill paths into\n the image.\n- Supports EvalScope native agents and external agent runners through the shared agent environment interface.\n- Saves verifier stdout, reward, and optional CTRF artifacts under the run output directory.\n\n## Evaluation Notes\n\n- Default `skill_mode` is `no-skill`.\n- Run `no-skill` and `with-skill` separately, then compare runs with EvalScope's run comparison tools.\n- `self-gen` and `tasks-extra` are not supported by this adapter version.\n- `runner='oracle'` runs the official `oracle/solve.sh` for smoke testing; agent runs require `agent_config`.\n- The verifier executes `verifier/test.sh` and may install dependencies from the network.\n\n## Scoring and Comparison\n\n- `score` is the verifier reward parsed from `/logs/verifier/reward.txt`.\n- `success` is derived locally as `1.0` when `score > 0`, otherwise `0.0`.\n- EvalScope does not automatically compute the no-skill versus with-skill delta; run both modes and compare their reports.\n", + "prompt_template": "{question}", + "system_prompt": "", + "few_shot_prompt_template": "", + "aggregation": "mean", + "extra_params": { + "tasks_dir": { + "type": "str", + "description": "Path to the SkillsBench tasks directory.", + "value": "" + }, + "task_ids": { + "type": "list", + "description": "Optional list of task ids to run.", + "value": [] + }, + "skill_mode": { + "type": "str", + "description": "SkillsBench skill mode.", + "value": "no-skill", + "choices": [ + "no-skill", + "with-skill" + ] + }, + "force_rebuild": { + "type": "bool", + "description": "Force rebuilding task Docker images.", + "value": false + }, + "agent_timeout_sec": { + "type": "float", + "description": "Override task agent timeout.", + "value": null + }, + "verifier_timeout_sec": { + "type": "float", + "description": "Override task verifier timeout.", + "value": null + }, + "runner": { + "type": "str", + "description": "Use \"oracle\" to run oracle/solve.sh instead of an agent.", + "value": "agent", + "choices": [ + "agent", + "oracle" + ] + } + }, + "sandbox_config": {}, + "category": "agent" + }, + "statistics": { + "total_samples": 0, + "subset_stats": [], + "prompt_length": { + "mean": 0, + "min": 0, + "max": 0, + "std": null + }, + "target_length_mean": null, + "computed_at": "2026-07-07T17:13:24.144419" + }, + "sample_example": {}, + "readme": { + "en": "# SkillsBench\n\n\n## Overview\n\nSkillsBench evaluates whether coding agents can discover and apply task-bundled Agent Skills. Each task contains an\ninstruction, an optional skill directory, a Docker environment, an oracle solution, and a verifier. EvalScope builds the\ntask Docker image, runs the selected agent or oracle in that image, then executes the task verifier.\n\n## Task Description\n\n- **Task Type**: Agent skill usage / tool-assisted task completion\n- **Input**: The natural-language task prompt from `task.md`\n- **Output**: Files or state changes produced inside the task container, scored by the task verifier\n- **Dataset**: Local SkillsBench task repository supplied through `extra_params.tasks_dir`\n- **Environment**: Per-task Docker image built from `environment/Dockerfile`\n- **Skills**: Optional task-bundled skills from `environment/skills`\n- **Metric**: `score` from `/logs/verifier/reward.txt`; `success` is 1 when `score > 0`\n\n## Key Features\n\n- Builds or reuses a content-hashed Docker image for each selected task.\n- Runs the task in `no-skill` or `with-skill` mode without mixing the two conditions in one EvalScope run.\n- Injects task-bundled skills through EvalScope's agent skill runtime instead of baking runner-specific skill paths into\n the image.\n- Supports EvalScope native agents and external agent runners through the shared agent environment interface.\n- Saves verifier stdout, reward, and optional CTRF artifacts under the run output directory.\n\n## Evaluation Notes\n\n- Default `skill_mode` is `no-skill`.\n- Run `no-skill` and `with-skill` separately, then compare runs with EvalScope's run comparison tools.\n- `self-gen` and `tasks-extra` are not supported by this adapter version.\n- `runner='oracle'` runs the official `oracle/solve.sh` for smoke testing; agent runs require `agent_config`.\n- The verifier executes `verifier/test.sh` and may install dependencies from the network.\n\n## Scoring and Comparison\n\n- `score` is the verifier reward parsed from `/logs/verifier/reward.txt`.\n- `success` is derived locally as `1.0` when `score > 0`, otherwise `0.0`.\n- EvalScope does not automatically compute the no-skill versus with-skill delta; run both modes and compare their reports.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `skillsbench` |\n| **Dataset ID** | `skillsbench` |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `MultiTurn` |\n| **Metrics** | `score` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `N/A` |\n\n\n## Data Statistics\n\n*Statistics not available.*\n\n## Sample Example\n\n*Sample example not available.*\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `tasks_dir` | `str` | `` | Path to the SkillsBench tasks directory. |\n| `task_ids` | `list` | `[]` | Optional list of task ids to run. |\n| `skill_mode` | `str` | `no-skill` | SkillsBench skill mode. Choices: ['no-skill', 'with-skill'] |\n| `force_rebuild` | `bool` | `False` | Force rebuilding task Docker images. |\n| `agent_timeout_sec` | `float` | `None` | Override task agent timeout. |\n| `verifier_timeout_sec` | `float` | `None` | Override task verifier timeout. |\n| `runner` | `str` | `agent` | Use \"oracle\" to run oracle/solve.sh instead of an agent. Choices: ['agent', 'oracle'] |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets skillsbench \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['skillsbench'],\n dataset_args={\n 'skillsbench': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# SkillsBench\n\n\n## 概述\n\nSkillsBench 用于评估编码智能体是否能够发现并应用任务捆绑的 Agent 技能。每个任务包含一条指令、一个可选的技能目录、一个 Docker 环境、一个参考解决方案（oracle solution）和一个验证器（verifier）。EvalScope 会构建任务的 Docker 镜像，在该镜像中运行所选的智能体或参考方案，然后执行任务验证器。\n\n## 任务描述\n\n- **任务类型**：智能体技能使用 / 工具辅助任务完成\n- **输入**：来自 `task.md` 的自然语言任务提示\n- **输出**：在任务容器内生成的文件或状态变更，由任务验证器评分\n- **数据集**：通过 `extra_params.tasks_dir` 提供的本地 SkillsBench 任务仓库\n- **环境**：基于 `environment/Dockerfile` 构建的每个任务专属 Docker 镜像\n- **技能**：可选的任务捆绑技能，位于 `environment/skills`\n- **指标**：来自 `/logs/verifier/reward.txt` 的 `score`；当 `score > 0` 时，`success` 为 1\n\n## 核心特性\n\n- 为每个选定任务构建或复用基于内容哈希的 Docker 镜像。\n- 在单次 EvalScope 运行中，以 `no-skill` 或 `with-skill` 模式运行任务，不会混合两种模式。\n- 通过 EvalScope 的智能体技能运行时注入任务捆绑技能，而非将运行器特定的技能路径硬编码进镜像。\n- 支持 EvalScope 原生智能体和通过共享智能体环境接口的外部智能体运行器。\n- 将验证器的标准输出、奖励值以及可选的 CTRF 工件保存至运行输出目录。\n\n## 评估说明\n\n- 默认 `skill_mode` 为 `no-skill`。\n- 分别运行 `no-skill` 和 `with-skill` 模式，然后使用 EvalScope 的运行对比工具进行比较。\n- 此适配器版本不支持 `self-gen` 和 `tasks-extra`。\n- `runner='oracle'` 会运行官方的 `oracle/solve.sh` 用于冒烟测试；智能体运行需提供 `agent_config`。\n- 验证器执行 `verifier/test.sh`，可能会从网络安装依赖项。\n\n## 评分与对比\n\n- `score` 是从 `/logs/verifier/reward.txt` 解析出的验证器奖励值。\n- `success` 在本地计算：当 `score > 0` 时为 `1.0`，否则为 `0.0`。\n- EvalScope 不会自动计算 `no-skill` 与 `with-skill` 的差异；需分别运行两种模式并对比其报告。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `skillsbench` |\n| **数据集ID** | `skillsbench` |\n| **论文** | N/A |\n| **标签** | `Agent`, `MultiTurn` |\n| **指标** | `score` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `N/A` |\n\n\n## 数据统计\n\n*统计数据不可用。*\n\n## 样例示例\n\n*样例示例不可用。*\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `tasks_dir` | `str` | `` | SkillsBench 任务目录的路径。 |\n| `task_ids` | `list` | `[]` | 可选的任务 ID 列表，用于指定运行哪些任务。 |\n| `skill_mode` | `str` | `no-skill` | SkillsBench 技能模式。选项：['no-skill', 'with-skill'] |\n| `force_rebuild` | `bool` | `False` | 强制重新构建任务 Docker 镜像。 |\n| `agent_timeout_sec` | `float` | `None` | 覆盖任务智能体的超时时间。 |\n| `verifier_timeout_sec` | `float` | `None` | 覆盖任务验证器的超时时间。 |\n| `runner` | `str` | `agent` | 使用 \"oracle\" 来运行 oracle/solve.sh 而非智能体。选项：['agent', 'oracle'] |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets skillsbench \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['skillsbench'],\n dataset_args={\n 'skillsbench': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "db5bef21e178e6e38a305be0faa35dc3", + "needs_translation": false + }, + "updated_at": "2026-07-07T17:13:24.145456", + "translation_updated_at": "2026-07-07T17:15:17" +} diff --git a/evalscope/evalscope/benchmarks/_meta/super_gpqa.json b/evalscope/evalscope/benchmarks/_meta/super_gpqa.json index ee7e383..5bde71c 100644 --- a/evalscope/evalscope/benchmarks/_meta/super_gpqa.json +++ b/evalscope/evalscope/benchmarks/_meta/super_gpqa.json @@ -799,4 +799,4 @@ }, "updated_at": "2026-01-28T17:31:32.461316", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/swe_bench_lite.json b/evalscope/evalscope/benchmarks/_meta/swe_bench_lite.json index 038531e..8373959 100644 --- a/evalscope/evalscope/benchmarks/_meta/swe_bench_lite.json +++ b/evalscope/evalscope/benchmarks/_meta/swe_bench_lite.json @@ -76,4 +76,4 @@ }, "updated_at": "2026-06-16T10:56:20.660351", "translation_updated_at": "2026-06-16T10:56:45" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/swe_bench_lite_agentic.json b/evalscope/evalscope/benchmarks/_meta/swe_bench_lite_agentic.json index eb2279e..94bf3af 100644 --- a/evalscope/evalscope/benchmarks/_meta/swe_bench_lite_agentic.json +++ b/evalscope/evalscope/benchmarks/_meta/swe_bench_lite_agentic.json @@ -15,31 +15,12 @@ "subset_list": [ "default" ], - "description": "\n## Overview\n\nSWE-bench Lite Agentic is the agentic-mode evaluation of SWE-bench Lite, a focused subset of SWE-bench containing 300 Issue-Pull Request pairs from 11 popular Python repositories. The model autonomously drives a multi-turn agent loop inside a per-instance Docker container to resolve real-world GitHub issues.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic)\n- **Input**: GitHub issue description (no oracle file context)\n- **Output**: Code patch (diff format) collected from `git diff` after autonomous editing\n- **Size**: 300 carefully selected test instances\n\n## Key Features\n\n- 300 test Issue-Pull Request pairs\n- 11 popular Python repositories covered\n- Real-world bugs with verified solutions\n- Multi-turn agent loop with per-instance Docker sandbox\n- More manageable than full SWE-bench while still challenging\n\n## Evaluation Notes\n\n- Requires `pip install swebench==4.1.0` before evaluation\n- Docker images are built/pulled automatically for each repository\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench.html) for detailed setup instructions\n- Popular benchmark variant for initial agentic model comparison\n\n## Agentic Mode\n\nThis benchmark drives a multi-turn agent loop (mirrors mini-swe-agent's\n`swebench.yaml`) inside a per-instance SWE-bench Docker container. The\nmodel issues `bash` commands to explore `/testbed`, edit source files,\nand finally submits its `git diff` patch by printing the sentinel\n`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` followed by the patch contents.\n\n`extra_params.action_protocol` selects between:\n- `toolcall` (default): OpenAI function-calling protocol with a single\n `bash` tool. Recommended for any model that supports tool calling.\n- `backticks`: text-based fallback expecting one\n ` ```mswea_bash_command ``` ` block per turn. For models without\n function-calling support.\n", + "description": "\n## Overview\n\nSWE-bench Lite Agentic is the agentic-mode evaluation of SWE-bench Lite, a focused subset of SWE-bench containing 300 Issue-Pull Request pairs from 11 popular Python repositories. The model autonomously drives a multi-turn agent loop inside a per-instance Docker container to resolve real-world GitHub issues.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic)\n- **Input**: GitHub issue description (no oracle file context)\n- **Output**: Code patch (diff format) collected from `git diff` after autonomous editing\n- **Size**: 300 carefully selected test instances\n\n## Key Features\n\n- 300 test Issue-Pull Request pairs\n- 11 popular Python repositories covered\n- Real-world bugs with verified solutions\n- Multi-turn agent loop with per-instance Docker sandbox\n- More manageable than full SWE-bench while still challenging\n\n## Evaluation Notes\n\n- Requires `pip install swebench==4.1.0` before evaluation\n- Docker images are built/pulled automatically for each repository\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench.html) for detailed setup instructions\n- Popular benchmark variant for initial agentic model comparison\n\n## Agentic Mode\n\nThis benchmark drives a multi-turn agent loop (mirrors mini-swe-agent's\n`swebench.yaml`) inside a per-instance SWE-bench Docker container. The\nmodel issues `bash` commands to explore `/testbed`, edit source files,\nand finally submits its `git diff` patch by printing the sentinel\n`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` followed by the patch contents.\n\nThe default `swe_bench_toolcall` strategy uses OpenAI function calling with\na single `bash` tool. Models without function-calling support can select\n`swe_bench_backticks` through `NativeAgentConfig.strategy`; that strategy\nexpects one ` ```mswea_bash_command ``` ` block per turn.\n", "prompt_template": "{question}", "system_prompt": "", "few_shot_prompt_template": "", "aggregation": "mean", "extra_params": { - "action_protocol": { - "type": "str", - "description": "Agent action protocol: \"toolcall\" (mainline OpenAI function-calling, mirrors mini-swe-agent swebench.yaml) or \"backticks\" (textbased mswea_bash_command fallback for models without function-calling support).", - "value": "toolcall", - "choices": [ - "toolcall", - "backticks" - ] - }, - "max_steps": { - "type": "int", - "description": "Maximum number of agent steps per sample.", - "value": 250 - }, - "command_timeout": { - "type": "float", - "description": "Default per-bash-command timeout in seconds.", - "value": 60.0 - }, "build_docker_images": { "type": "bool", "description": "Build Docker images locally for each sample.", @@ -67,6 +48,10 @@ } }, "sandbox_config": {}, + "agent_config": { + "strategy": "swe_bench_toolcall", + "max_steps": 250 + }, "category": "agent" }, "statistics": { @@ -158,11 +143,11 @@ "truncated": false }, "readme": { - "en": "# SWE-bench_Lite_Agentic\n\n\n## Overview\n\nSWE-bench Lite Agentic is the agentic-mode evaluation of SWE-bench Lite, a focused subset of SWE-bench containing 300 Issue-Pull Request pairs from 11 popular Python repositories. The model autonomously drives a multi-turn agent loop inside a per-instance Docker container to resolve real-world GitHub issues.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic)\n- **Input**: GitHub issue description (no oracle file context)\n- **Output**: Code patch (diff format) collected from `git diff` after autonomous editing\n- **Size**: 300 carefully selected test instances\n\n## Key Features\n\n- 300 test Issue-Pull Request pairs\n- 11 popular Python repositories covered\n- Real-world bugs with verified solutions\n- Multi-turn agent loop with per-instance Docker sandbox\n- More manageable than full SWE-bench while still challenging\n\n## Evaluation Notes\n\n- Requires `pip install swebench==4.1.0` before evaluation\n- Docker images are built/pulled automatically for each repository\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench.html) for detailed setup instructions\n- Popular benchmark variant for initial agentic model comparison\n\n## Agentic Mode\n\nThis benchmark drives a multi-turn agent loop (mirrors mini-swe-agent's\n`swebench.yaml`) inside a per-instance SWE-bench Docker container. The\nmodel issues `bash` commands to explore `/testbed`, edit source files,\nand finally submits its `git diff` patch by printing the sentinel\n`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` followed by the patch contents.\n\n`extra_params.action_protocol` selects between:\n- `toolcall` (default): OpenAI function-calling protocol with a single\n `bash` tool. Recommended for any model that supports tool calling.\n- `backticks`: text-based fallback expecting one\n ` ```mswea_bash_command ``` ` block per turn. For models without\n function-calling support.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `swe_bench_lite_agentic` |\n| **Dataset ID** | [princeton-nlp/SWE-bench_Lite](https://modelscope.cn/datasets/princeton-nlp/SWE-bench_Lite/summary) |\n| **Paper** | N/A |\n| **Tags** | `Coding` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 300 |\n| Prompt Length (Mean) | 1661.18 chars |\n| Prompt Length (Min/Max) | 230 / 24770 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"c8f45390\",\n \"content\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matri ... [TRUNCATED 762 chars] ... [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"problem_statement\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matri ... [TRUNCATED 762 chars] ... [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\",\n \"instance_id\": \"astropy__astropy-12907\",\n \"base_commit\": \"d16bfe05a744909de4b27f5875fe0d4ed41ce607\",\n \"patch\": \"diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\\n--- a/astropy/modeling/separable.py\\n+++ b/astropy/modeling/separable.py\\n@@ -242,7 +242,7 @@ def _cstack(left, right):\\n cright = _coord_matrix(right, 'right', noutp)\\n else:\\n cright = np.zeros((noutp, right.shape[1]))\\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\\n \\n return np.hstack([cleft, cright])\\n \\n\",\n \"PASS_TO_PASS\": [\n \"astropy/modeling/tests/test_separable.py::test_coord_matrix\",\n \"astropy/modeling/tests/test_separable.py::test_cdot\",\n \"astropy/modeling/tests/test_separable.py::test_cstack\",\n \"astropy/modeling/tests/test_separable.py::test_arith_oper\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model0-result0]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model1-result1]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model2-result2]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model3-result3]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model4-result4]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model5-result5]\",\n \"... [TRUNCATED 3 more items] ...\"\n ],\n \"FAIL_TO_PASS\": [\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model6-result6]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model9-result9]\"\n ],\n \"test_patch\": \"diff --git a/astropy/modeling/tests/test_separable.py b/astropy/modeling/tests/test_separable.py\\n--- a/astropy/modeling/tests/test_separable.py\\n+++ b/astropy/modeling/tests/test_separable.py\\n@@ -28,6 +28,13 @@\\n p1 = models.Polynomial1D(1, nam ... [TRUNCATED 931 chars] ... [True, True, False, False, False],\\n+ [False, False, True, False, False],\\n+ [False, False, False, True, False],\\n+ [False, False, False, False, True]]))),\\n }\\n \\n \\n\",\n \"version\": \"4.3\",\n \"repo\": \"astropy/astropy\",\n \"environment_setup_commit\": \"298ccb478e6bf092953bca67a3d29dc6c35f6752\",\n \"hints_text\": \"\",\n \"created_at\": \"2022-03-03T15:14:54Z\",\n \"docker_image\": \"swebench/sweb.eval.arm64.astropy_1776_astropy-12907:latest\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `action_protocol` | `str` | `toolcall` | Agent action protocol: \"toolcall\" (mainline OpenAI function-calling, mirrors mini-swe-agent swebench.yaml) or \"backticks\" (textbased mswea_bash_command fallback for models without function-calling support). Choices: ['toolcall', 'backticks'] |\n| `max_steps` | `int` | `250` | Maximum number of agent steps per sample. |\n| `command_timeout` | `float` | `60.0` | Default per-bash-command timeout in seconds. |\n| `build_docker_images` | `bool` | `True` | Build Docker images locally for each sample. |\n| `pull_remote_images_if_available` | `bool` | `True` | Attempt to pull existing remote Docker images before building. |\n| `force_arch` | `str` | `` | Optionally force a specific architecture for image build/pull. Choices: ['', 'arm64', 'x86_64'] |\n| `dockerhub_username` | `str` | `swebench` | DockerHub user/org namespace for remote SWE-bench images. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_lite_agentic \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_lite_agentic'],\n dataset_args={\n 'swe_bench_lite_agentic': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# SWE-bench_Lite_Agentic\n\n\n## 概述\n\nSWE-bench Lite Agentic 是 SWE-bench Lite 的代理模式（agentic-mode）评估版本。SWE-bench Lite 是 SWE-bench 的一个精选子集，包含来自 11 个热门 Python 仓库的 300 个 Issue-Pull Request 对。模型在每个实例独立的 Docker 容器中自主驱动多轮代理循环，以解决真实的 GitHub 问题。\n\n## 任务描述\n\n- **任务类型**：自动化软件工程 / 缺陷修复（代理模式）\n- **输入**：GitHub issue 描述（不提供 oracle 文件上下文）\n- **输出**：通过 `git diff` 收集的代码补丁（diff 格式），由模型自主编辑后生成\n- **规模**：300 个精心挑选的测试实例\n\n## 主要特性\n\n- 包含 300 个测试用的 Issue-Pull Request 对\n- 覆盖 11 个热门 Python 仓库\n- 真实世界中的缺陷，且已有验证过的解决方案\n- 每个实例使用独立的 Docker 沙箱环境，支持多轮代理循环\n- 相比完整版 SWE-bench 更易于管理，但仍具挑战性\n\n## 评估说明\n\n- 评估前需先执行 `pip install swebench==4.1.0`\n- 每个仓库的 Docker 镜像会自动构建或拉取\n- 详细设置说明请参阅 [使用文档](https://evalscope.readthedocs.io/zh-cn/latest/third_party/swe_bench.html)\n- 此基准测试是初始代理模型对比的常用变体\n\n## 代理模式\n\n该基准测试在每个实例专属的 SWE-bench Docker 容器内驱动一个多轮代理循环（与 mini-swe-agent 的 `swebench.yaml` 配置一致）。模型通过发出 `bash` 命令来探索 `/testbed` 目录、编辑源文件，并最终通过打印哨兵字符串 `COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` 及其后的补丁内容来提交 `git diff` 补丁。\n\n`extra_params.action_protocol` 可选择以下两种协议：\n- `toolcall`（默认）：采用 OpenAI 函数调用协议，仅提供一个 `bash` 工具。推荐用于支持工具调用的模型。\n- `backticks`：基于文本的备用方案，每轮期望一个 ` ```mswea_bash_command ``` ` 代码块。适用于不支持函数调用的模型。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `swe_bench_lite_agentic` |\n| **数据集 ID** | [princeton-nlp/SWE-bench_Lite](https://modelscope.cn/datasets/princeton-nlp/SWE-bench_Lite/summary) |\n| **论文** | N/A |\n| **标签** | `Coding` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 300 |\n| 提示词长度（平均） | 1661.18 字符 |\n| 提示词长度（最小/最大） | 230 / 24770 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"c8f45390\",\n \"content\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matri ... [TRUNCATED 762 chars] ... [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"problem_statement\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matri ... [TRUNCATED 762 chars] ... [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\",\n \"instance_id\": \"astropy__astropy-12907\",\n \"base_commit\": \"d16bfe05a744909de4b27f5875fe0d4ed41ce607\",\n \"patch\": \"diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\\n--- a/astropy/modeling/separable.py\\n+++ b/astropy/modeling/separable.py\\n@@ -242,7 +242,7 @@ def _cstack(left, right):\\n cright = _coord_matrix(right, 'right', noutp)\\n else:\\n cright = np.zeros((noutp, right.shape[1]))\\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\\n \\n return np.hstack([cleft, cright])\\n \\n\",\n \"PASS_TO_PASS\": [\n \"astropy/modeling/tests/test_separable.py::test_coord_matrix\",\n \"astropy/modeling/tests/test_separable.py::test_cdot\",\n \"astropy/modeling/tests/test_separable.py::test_cstack\",\n \"astropy/modeling/tests/test_separable.py::test_arith_oper\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model0-result0]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model1-result1]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model2-result2]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model3-result3]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model4-result4]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model5-result5]\",\n \"... [TRUNCATED 3 more items] ...\"\n ],\n \"FAIL_TO_PASS\": [\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model6-result6]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model9-result9]\"\n ],\n \"test_patch\": \"diff --git a/astropy/modeling/tests/test_separable.py b/astropy/modeling/tests/test_separable.py\\n--- a/astropy/modeling/tests/test_separable.py\\n+++ b/astropy/modeling/tests/test_separable.py\\n@@ -28,6 +28,13 @@\\n p1 = models.Polynomial1D(1, nam ... [TRUNCATED 931 chars] ... [True, True, False, False, False],\\n+ [False, False, True, False, False],\\n+ [False, False, False, True, False],\\n+ [False, False, False, False, True]]))),\\n }\\n \\n \\n\",\n \"version\": \"4.3\",\n \"repo\": \"astropy/astropy\",\n \"environment_setup_commit\": \"298ccb478e6bf092953bca67a3d29dc6c35f6752\",\n \"hints_text\": \"\",\n \"created_at\": \"2022-03-03T15:14:54Z\",\n \"docker_image\": \"swebench/sweb.eval.arm64.astropy_1776_astropy-12907:latest\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `action_protocol` | `str` | `toolcall` | 代理动作协议：\"toolcall\"（主流 OpenAI 函数调用方式，与 mini-swe-agent 的 swebench.yaml 一致）或 \"backticks\"（基于文本的 mswea_bash_command 回退方案，适用于不支持函数调用的模型）。可选项：['toolcall', 'backticks'] |\n| `max_steps` | `int` | `250` | 每个样本的最大代理步数。 |\n| `command_timeout` | `float` | `60.0` | 每个 bash 命令的默认超时时间（秒）。 |\n| `build_docker_images` | `bool` | `True` | 为每个样本本地构建 Docker 镜像。 |\n| `pull_remote_images_if_available` | `bool` | `True` | 在构建前尝试拉取已存在的远程 Docker 镜像。 |\n| `force_arch` | `str` | `` | 可选地强制指定镜像构建/拉取的目标架构。可选项：['', 'arm64', 'x86_64'] |\n| `dockerhub_username` | `str` | `swebench` | 远程 SWE-bench 镜像所用的 DockerHub 用户/组织命名空间。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_lite_agentic \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_lite_agentic'],\n dataset_args={\n 'swe_bench_lite_agentic': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "6ae7ca1efa52cbabb670457dbff9350c", + "en": "# SWE-bench_Lite_Agentic\n\n\n## Overview\n\nSWE-bench Lite Agentic is the agentic-mode evaluation of SWE-bench Lite, a focused subset of SWE-bench containing 300 Issue-Pull Request pairs from 11 popular Python repositories. The model autonomously drives a multi-turn agent loop inside a per-instance Docker container to resolve real-world GitHub issues.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic)\n- **Input**: GitHub issue description (no oracle file context)\n- **Output**: Code patch (diff format) collected from `git diff` after autonomous editing\n- **Size**: 300 carefully selected test instances\n\n## Key Features\n\n- 300 test Issue-Pull Request pairs\n- 11 popular Python repositories covered\n- Real-world bugs with verified solutions\n- Multi-turn agent loop with per-instance Docker sandbox\n- More manageable than full SWE-bench while still challenging\n\n## Evaluation Notes\n\n- Requires `pip install swebench==4.1.0` before evaluation\n- Docker images are built/pulled automatically for each repository\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench.html) for detailed setup instructions\n- Popular benchmark variant for initial agentic model comparison\n\n## Agentic Mode\n\nThis benchmark drives a multi-turn agent loop (mirrors mini-swe-agent's\n`swebench.yaml`) inside a per-instance SWE-bench Docker container. The\nmodel issues `bash` commands to explore `/testbed`, edit source files,\nand finally submits its `git diff` patch by printing the sentinel\n`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` followed by the patch contents.\n\nThe default `swe_bench_toolcall` strategy uses OpenAI function calling with\na single `bash` tool. Models without function-calling support can select\n`swe_bench_backticks` through `NativeAgentConfig.strategy`; that strategy\nexpects one ` ```mswea_bash_command ``` ` block per turn.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `swe_bench_lite_agentic` |\n| **Dataset ID** | [princeton-nlp/SWE-bench_Lite](https://modelscope.cn/datasets/princeton-nlp/SWE-bench_Lite/summary) |\n| **Paper** | N/A |\n| **Tags** | `Coding` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 300 |\n| Prompt Length (Mean) | 1661.18 chars |\n| Prompt Length (Min/Max) | 230 / 24770 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"c8f45390\",\n \"content\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matri ... [TRUNCATED 762 chars] ... [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"problem_statement\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matri ... [TRUNCATED 762 chars] ... [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\",\n \"instance_id\": \"astropy__astropy-12907\",\n \"base_commit\": \"d16bfe05a744909de4b27f5875fe0d4ed41ce607\",\n \"patch\": \"diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\\n--- a/astropy/modeling/separable.py\\n+++ b/astropy/modeling/separable.py\\n@@ -242,7 +242,7 @@ def _cstack(left, right):\\n cright = _coord_matrix(right, 'right', noutp)\\n else:\\n cright = np.zeros((noutp, right.shape[1]))\\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\\n \\n return np.hstack([cleft, cright])\\n \\n\",\n \"PASS_TO_PASS\": [\n \"astropy/modeling/tests/test_separable.py::test_coord_matrix\",\n \"astropy/modeling/tests/test_separable.py::test_cdot\",\n \"astropy/modeling/tests/test_separable.py::test_cstack\",\n \"astropy/modeling/tests/test_separable.py::test_arith_oper\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model0-result0]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model1-result1]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model2-result2]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model3-result3]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model4-result4]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model5-result5]\",\n \"... [TRUNCATED 3 more items] ...\"\n ],\n \"FAIL_TO_PASS\": [\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model6-result6]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model9-result9]\"\n ],\n \"test_patch\": \"diff --git a/astropy/modeling/tests/test_separable.py b/astropy/modeling/tests/test_separable.py\\n--- a/astropy/modeling/tests/test_separable.py\\n+++ b/astropy/modeling/tests/test_separable.py\\n@@ -28,6 +28,13 @@\\n p1 = models.Polynomial1D(1, nam ... [TRUNCATED 931 chars] ... [True, True, False, False, False],\\n+ [False, False, True, False, False],\\n+ [False, False, False, True, False],\\n+ [False, False, False, False, True]]))),\\n }\\n \\n \\n\",\n \"version\": \"4.3\",\n \"repo\": \"astropy/astropy\",\n \"environment_setup_commit\": \"298ccb478e6bf092953bca67a3d29dc6c35f6752\",\n \"hints_text\": \"\",\n \"created_at\": \"2022-03-03T15:14:54Z\",\n \"docker_image\": \"swebench/sweb.eval.arm64.astropy_1776_astropy-12907:latest\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `build_docker_images` | `bool` | `True` | Build Docker images locally for each sample. |\n| `pull_remote_images_if_available` | `bool` | `True` | Attempt to pull existing remote Docker images before building. |\n| `force_arch` | `str` | `` | Optionally force a specific architecture for image build/pull. Choices: ['', 'arm64', 'x86_64'] |\n| `dockerhub_username` | `str` | `swebench` | DockerHub user/org namespace for remote SWE-bench images. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_lite_agentic \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"swe_bench_toolcall\",\"max_steps\":250}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_lite_agentic'],\n agent_config=NativeAgentConfig(\n strategy='swe_bench_toolcall',\n max_steps=250,\n ),\n dataset_args={\n 'swe_bench_lite_agentic': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# SWE-bench_Lite_Agentic\n\n\n## 概述\n\nSWE-bench Lite Agentic 是 SWE-bench Lite 的代理模式（agentic-mode）评估版本。SWE-bench Lite 是 SWE-bench 的一个精选子集，包含来自 11 个热门 Python 仓库的 300 个 Issue-Pull Request 对。模型在每个实例独立的 Docker 容器中自主驱动多轮代理循环，以解决真实的 GitHub 问题。\n\n## 任务描述\n\n- **任务类型**：自动化软件工程 / 缺陷修复（代理模式）\n- **输入**：GitHub issue 描述（不包含 oracle 文件上下文）\n- **输出**：通过 `git diff` 收集的代码补丁（diff 格式），由模型自主编辑后生成\n- **规模**：300 个精心挑选的测试实例\n\n## 主要特性\n\n- 包含 300 个测试用的 Issue-Pull Request 对\n- 覆盖 11 个热门 Python 仓库\n- 真实世界中的缺陷，且已有验证过的解决方案\n- 每个实例均在独立的 Docker 沙箱中运行多轮代理循环\n- 相比完整版 SWE-bench 更易管理，但仍具挑战性\n\n## 评估说明\n\n- 评估前需先执行 `pip install swebench==4.1.0`\n- 每个仓库的 Docker 镜像会自动构建或拉取\n- 详细设置说明请参阅 [使用文档](https://evalscope.readthedocs.io/zh-cn/latest/third_party/swe_bench.html)\n- 是用于初步比较代理模型性能的常用基准变体\n\n## 代理模式\n\n该基准在每个实例专属的 SWE-bench Docker 容器内驱动一个多轮代理循环（与 mini-swe-agent 的 `swebench.yaml` 配置一致）。模型通过发出 `bash` 命令来探索 `/testbed` 目录、编辑源文件，并最终通过打印哨兵字符串 `COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` 及其后的补丁内容来提交 `git diff` 补丁。\n\n默认的 `swe_bench_toolcall` 策略使用 OpenAI 函数调用，仅提供一个 `bash` 工具。对于不支持函数调用的模型，可通过 `NativeAgentConfig.strategy` 选择 `swe_bench_backticks` 策略；该策略要求每轮输出中包含一个 ` ```mswea_bash_command ``` ` 代码块。\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `swe_bench_lite_agentic` |\n| **数据集ID** | [princeton-nlp/SWE-bench_Lite](https://modelscope.cn/datasets/princeton-nlp/SWE-bench_Lite/summary) |\n| **论文** | N/A |\n| **标签** | `Coding` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估分割** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 300 |\n| 提示词长度（平均） | 1661.18 字符 |\n| 提示词长度（最小/最大） | 230 / 24770 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"c8f45390\",\n \"content\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matri ... [TRUNCATED 762 chars] ... [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"problem_statement\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matri ... [TRUNCATED 762 chars] ... [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\",\n \"instance_id\": \"astropy__astropy-12907\",\n \"base_commit\": \"d16bfe05a744909de4b27f5875fe0d4ed41ce607\",\n \"patch\": \"diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\\n--- a/astropy/modeling/separable.py\\n+++ b/astropy/modeling/separable.py\\n@@ -242,7 +242,7 @@ def _cstack(left, right):\\n cright = _coord_matrix(right, 'right', noutp)\\n else:\\n cright = np.zeros((noutp, right.shape[1]))\\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\\n \\n return np.hstack([cleft, cright])\\n \\n\",\n \"PASS_TO_PASS\": [\n \"astropy/modeling/tests/test_separable.py::test_coord_matrix\",\n \"astropy/modeling/tests/test_separable.py::test_cdot\",\n \"astropy/modeling/tests/test_separable.py::test_cstack\",\n \"astropy/modeling/tests/test_separable.py::test_arith_oper\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model0-result0]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model1-result1]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model2-result2]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model3-result3]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model4-result4]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model5-result5]\",\n \"... [TRUNCATED 3 more items] ...\"\n ],\n \"FAIL_TO_PASS\": [\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model6-result6]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model9-result9]\"\n ],\n \"test_patch\": \"diff --git a/astropy/modeling/tests/test_separable.py b/astropy/modeling/tests/test_separable.py\\n--- a/astropy/modeling/tests/test_separable.py\\n+++ b/astropy/modeling/tests/test_separable.py\\n@@ -28,6 +28,13 @@\\n p1 = models.Polynomial1D(1, nam ... [TRUNCATED 931 chars] ... [True, True, False, False, False],\\n+ [False, False, True, False, False],\\n+ [False, False, False, True, False],\\n+ [False, False, False, False, True]]))),\\n }\\n \\n \\n\",\n \"version\": \"4.3\",\n \"repo\": \"astropy/astropy\",\n \"environment_setup_commit\": \"298ccb478e6bf092953bca67a3d29dc6c35f6752\",\n \"hints_text\": \"\",\n \"created_at\": \"2022-03-03T15:14:54Z\",\n \"docker_image\": \"swebench/sweb.eval.arm64.astropy_1776_astropy-12907:latest\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `build_docker_images` | `bool` | `True` | 为每个样本在本地构建 Docker 镜像。 |\n| `pull_remote_images_if_available` | `bool` | `True` | 在构建前尝试拉取已存在的远程 Docker 镜像。 |\n| `force_arch` | `str` | `` | 可选地强制指定镜像构建/拉取的架构。选项：['', 'arm64', 'x86_64'] |\n| `dockerhub_username` | `str` | `swebench` | 远程 SWE-bench 镜像在 DockerHub 上的用户/组织命名空间。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_lite_agentic \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"swe_bench_toolcall\",\"max_steps\":250}' \\\n --limit 10 # 正式评估时请移除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_lite_agentic'],\n agent_config=NativeAgentConfig(\n strategy='swe_bench_toolcall',\n max_steps=250,\n ),\n dataset_args={\n 'swe_bench_lite_agentic': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请移除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "a843becfdd38e16bc2cc9ec1f43fa6c6", "needs_translation": false }, - "updated_at": "2026-06-16T10:56:20.669937", - "translation_updated_at": "2026-06-16T10:56:45" -} \ No newline at end of file + "updated_at": "2026-07-10T20:24:34.529747", + "translation_updated_at": "2026-07-10T20:24:42" +} diff --git a/evalscope/evalscope/benchmarks/_meta/swe_bench_multilingual_agentic.json b/evalscope/evalscope/benchmarks/_meta/swe_bench_multilingual_agentic.json index 48f5d0d..49a7e0f 100644 --- a/evalscope/evalscope/benchmarks/_meta/swe_bench_multilingual_agentic.json +++ b/evalscope/evalscope/benchmarks/_meta/swe_bench_multilingual_agentic.json @@ -15,31 +15,12 @@ "subset_list": [ "default" ], - "description": "\n## Overview\n\nSWE-bench Multilingual Agentic is the agentic-mode evaluation of SWE-bench Multilingual, a 300-task SWE-bench-style benchmark spanning 42 repositories and 9 programming languages. The model autonomously explores, edits, and submits a patch through a multi-turn agent loop inside a per-instance Docker container.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic, Multilingual)\n- **Input**: GitHub issue description (no oracle file context)\n- **Output**: Code patch (diff format) collected from `git diff` after autonomous editing\n- **Languages**: C, C++, Go, Java, JavaScript/TypeScript, PHP, Ruby, and Rust\n\n## Key Features\n\n- 300 curated Issue-Pull Request tasks\n- 42 real-world repositories across 9 programming languages\n- Multi-turn agent loop with per-instance SWE-bench Docker sandbox\n- SWE-bench-compatible patch evaluation using fail-to-pass and pass-to-pass tests\n\n## Evaluation Notes\n\n- Requires `pip install swebench==4.1.0` before evaluation\n- Uses the official SWE-bench Multilingual x86_64 instance images and sets Docker platform to `linux/amd64` automatically\n- Docker images are built/pulled automatically for each instance\n- Timeout of 1800 seconds (30 min) per instance for final patch validation\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench.html) for detailed setup instructions\n- Supports both local image building and remote image pulling\n\n## Agentic Mode\n\nThis benchmark drives a multi-turn agent loop (mirrors mini-swe-agent's\n`swebench.yaml`) inside a per-instance SWE-bench Docker container. The\nmodel issues `bash` commands to explore `/testbed`, edit source files,\nand finally submits its `git diff` patch by printing the sentinel\n`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` followed by the patch contents.\n\n`extra_params.action_protocol` selects between:\n- `toolcall` (default): OpenAI function-calling protocol with a single\n `bash` tool. Recommended for any model that supports tool calling.\n- `backticks`: text-based fallback expecting one\n ` ```mswea_bash_command ``` ` block per turn. For models without\n function-calling support.\n", + "description": "\n## Overview\n\nSWE-bench Multilingual Agentic is the agentic-mode evaluation of SWE-bench Multilingual, a 300-task SWE-bench-style benchmark spanning 42 repositories and 9 programming languages. The model autonomously explores, edits, and submits a patch through a multi-turn agent loop inside a per-instance Docker container.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic, Multilingual)\n- **Input**: GitHub issue description (no oracle file context)\n- **Output**: Code patch (diff format) collected from `git diff` after autonomous editing\n- **Languages**: C, C++, Go, Java, JavaScript/TypeScript, PHP, Ruby, and Rust\n\n## Key Features\n\n- 300 curated Issue-Pull Request tasks\n- 42 real-world repositories across 9 programming languages\n- Multi-turn agent loop with per-instance SWE-bench Docker sandbox\n- SWE-bench-compatible patch evaluation using fail-to-pass and pass-to-pass tests\n\n## Evaluation Notes\n\n- Requires `pip install swebench==4.1.0` before evaluation\n- Uses the official SWE-bench Multilingual x86_64 instance images and sets Docker platform to `linux/amd64` automatically\n- Docker images are built/pulled automatically for each instance\n- Timeout of 1800 seconds (30 min) per instance for final patch validation\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench.html) for detailed setup instructions\n- Supports both local image building and remote image pulling\n\n## Agentic Mode\n\nThis benchmark drives a multi-turn agent loop (mirrors mini-swe-agent's\n`swebench.yaml`) inside a per-instance SWE-bench Docker container. The\nmodel issues `bash` commands to explore `/testbed`, edit source files,\nand finally submits its `git diff` patch by printing the sentinel\n`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` followed by the patch contents.\n\nThe default `swe_bench_toolcall` strategy uses OpenAI function calling with\na single `bash` tool. Models without function-calling support can select\n`swe_bench_backticks` through `NativeAgentConfig.strategy`; that strategy\nexpects one ` ```mswea_bash_command ``` ` block per turn.\n", "prompt_template": "{question}", "system_prompt": "", "few_shot_prompt_template": "", "aggregation": "mean", "extra_params": { - "action_protocol": { - "type": "str", - "description": "Agent action protocol: \"toolcall\" (mainline OpenAI function-calling, mirrors mini-swe-agent swebench.yaml) or \"backticks\" (textbased mswea_bash_command fallback for models without function-calling support).", - "value": "toolcall", - "choices": [ - "toolcall", - "backticks" - ] - }, - "max_steps": { - "type": "int", - "description": "Maximum number of agent steps per sample.", - "value": 250 - }, - "command_timeout": { - "type": "float", - "description": "Default per-bash-command timeout in seconds.", - "value": 60.0 - }, "build_docker_images": { "type": "bool", "description": "Build Docker images locally for each sample.", @@ -67,6 +48,10 @@ } }, "sandbox_config": {}, + "agent_config": { + "strategy": "swe_bench_toolcall", + "max_steps": 250 + }, "category": "agent" }, "statistics": { @@ -148,11 +133,11 @@ "truncated": false }, "readme": { - "en": "# SWE-bench_Multilingual_Agentic\n\n\n## Overview\n\nSWE-bench Multilingual Agentic is the agentic-mode evaluation of SWE-bench Multilingual, a 300-task SWE-bench-style benchmark spanning 42 repositories and 9 programming languages. The model autonomously explores, edits, and submits a patch through a multi-turn agent loop inside a per-instance Docker container.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic, Multilingual)\n- **Input**: GitHub issue description (no oracle file context)\n- **Output**: Code patch (diff format) collected from `git diff` after autonomous editing\n- **Languages**: C, C++, Go, Java, JavaScript/TypeScript, PHP, Ruby, and Rust\n\n## Key Features\n\n- 300 curated Issue-Pull Request tasks\n- 42 real-world repositories across 9 programming languages\n- Multi-turn agent loop with per-instance SWE-bench Docker sandbox\n- SWE-bench-compatible patch evaluation using fail-to-pass and pass-to-pass tests\n\n## Evaluation Notes\n\n- Requires `pip install swebench==4.1.0` before evaluation\n- Uses the official SWE-bench Multilingual x86_64 instance images and sets Docker platform to `linux/amd64` automatically\n- Docker images are built/pulled automatically for each instance\n- Timeout of 1800 seconds (30 min) per instance for final patch validation\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench.html) for detailed setup instructions\n- Supports both local image building and remote image pulling\n\n## Agentic Mode\n\nThis benchmark drives a multi-turn agent loop (mirrors mini-swe-agent's\n`swebench.yaml`) inside a per-instance SWE-bench Docker container. The\nmodel issues `bash` commands to explore `/testbed`, edit source files,\nand finally submits its `git diff` patch by printing the sentinel\n`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` followed by the patch contents.\n\n`extra_params.action_protocol` selects between:\n- `toolcall` (default): OpenAI function-calling protocol with a single\n `bash` tool. Recommended for any model that supports tool calling.\n- `backticks`: text-based fallback expecting one\n ` ```mswea_bash_command ``` ` block per turn. For models without\n function-calling support.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `swe_bench_multilingual_agentic` |\n| **Dataset ID** | [SWE-bench/SWE-bench_Multilingual](https://modelscope.cn/datasets/SWE-bench/SWE-bench_Multilingual/summary) |\n| **Paper** | N/A |\n| **Tags** | `Coding` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 300 |\n| Prompt Length (Mean) | 2197.94 chars |\n| Prompt Length (Min/Max) | 124 / 69351 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"52ece3cf\",\n \"content\": \"Support Post aggregation function pow(f1,f2) to cater for square, cube , square root.\\n### Description\\r\\n\\r\\nPlease describe the feature or change with as much detail as possible. \\r\\n\\r\\nAs of now the only supported arithmetic functions are +, -, *, ... [TRUNCATED 284 chars] ... \\r\\n\\r\\nThe proposal is to add a `pow` function which enables all the about usecase . Square of a number can be represent by pow(f1,2) , Cube can be represented as power(f1 ,3) , Squar root of a number can be represented by power(f1,0.5) ,\\r\\n\\r\\n\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"problem_statement\": \"Support Post aggregation function pow(f1,f2) to cater for square, cube , square root.\\n### Description\\r\\n\\r\\nPlease describe the feature or change with as much detail as possible. \\r\\n\\r\\nAs of now the only supported arithmetic functions are +, -, *, ... [TRUNCATED 284 chars] ... \\r\\n\\r\\nThe proposal is to add a `pow` function which enables all the about usecase . Square of a number can be represent by pow(f1,2) , Cube can be represented as power(f1 ,3) , Squar root of a number can be represented by power(f1,0.5) ,\\r\\n\\r\\n\\n\",\n \"instance_id\": \"apache__druid-13704\",\n \"base_commit\": \"51dfde02840017092486fb75be2b16566aff6a19\",\n \"patch\": \"diff --git a/docs/querying/post-aggregations.md b/docs/querying/post-aggregations.md\\nindex c75d122eb20a..935ca8fbce16 100644\\n--- a/docs/querying/post-aggregations.md\\n+++ b/docs/querying/post-aggregations.md\\n@@ -36,7 +36,7 @@ There are several ... [TRUNCATED 1110 chars] ... }\\n+ },\\n+\\n+ POW(\\\"pow\\\") {\\n+ @Override\\n+ public double compute(double lhs, double rhs)\\n+ {\\n+ return Math.pow(lhs, rhs);\\n+ }\\n };\\n \\n private static final Map LOOKUP_MAP = new HashMap<>();\\n\",\n \"PASS_TO_PASS\": [\n \"org.apache.druid.query.aggregation.post.ArithmeticPostAggregatorTest#testDiv\",\n \"org.apache.druid.query.aggregation.post.ArithmeticPostAggregatorTest#testQuotient\"\n ],\n \"FAIL_TO_PASS\": [\n \"org.apache.druid.query.aggregation.post.ArithmeticPostAggregatorTest#testPow\"\n ],\n \"test_patch\": \"diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java\\nindex a93034427539..7e1d4d112 ... [TRUNCATED 1358 chars] ... s(1.0, agg.compute(ImmutableMap.of(\\\"value\\\", 1)));\\n+ Assert.assertEquals(1.0, agg.compute(ImmutableMap.of(\\\"value\\\", -1)));\\n+ Assert.assertEquals(1.0, agg.compute(ImmutableMap.of(\\\"value\\\", .5)));\\n+ }\\n @Test\\n public void testDiv()\\n {\\n\",\n \"version\": \"13704\",\n \"repo\": \"apache/druid\",\n \"environment_setup_commit\": null,\n \"hints_text\": \"\",\n \"created_at\": \"2023-01-23 04:10:47\",\n \"docker_image\": \"swebench/sweb.eval.x86_64.apache_1776_druid-13704:latest\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `action_protocol` | `str` | `toolcall` | Agent action protocol: \"toolcall\" (mainline OpenAI function-calling, mirrors mini-swe-agent swebench.yaml) or \"backticks\" (textbased mswea_bash_command fallback for models without function-calling support). Choices: ['toolcall', 'backticks'] |\n| `max_steps` | `int` | `250` | Maximum number of agent steps per sample. |\n| `command_timeout` | `float` | `60.0` | Default per-bash-command timeout in seconds. |\n| `build_docker_images` | `bool` | `True` | Build Docker images locally for each sample. |\n| `pull_remote_images_if_available` | `bool` | `True` | Attempt to pull existing remote Docker images before building. |\n| `force_arch` | `str` | `` | Optionally force a specific architecture for image build/pull. Choices: ['', 'arm64', 'x86_64'] |\n| `dockerhub_username` | `str` | `swebench` | DockerHub user/org namespace for remote SWE-bench images. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_multilingual_agentic \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_multilingual_agentic'],\n dataset_args={\n 'swe_bench_multilingual_agentic': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# SWE-bench_Multilingual_Agentic\n\n\n## 概述\n\nSWE-bench Multilingual Agentic 是 SWE-bench Multilingual 的代理模式（agentic-mode）评估版本。该基准测试包含 300 个 SWE-bench 风格的任务，涵盖 42 个代码仓库和 9 种编程语言。模型在每个实例专属的 Docker 容器中，通过多轮代理循环自主探索、编辑代码并提交补丁。\n\n## 任务描述\n\n- **任务类型**：自动化软件工程 / 缺陷修复（代理式、多语言）\n- **输入**：GitHub issue 描述（不提供 oracle 文件上下文）\n- **输出**：模型自主编辑后通过 `git diff` 生成的代码补丁（diff 格式）\n- **支持语言**：C、C++、Go、Java、JavaScript/TypeScript、PHP、Ruby 和 Rust\n\n## 主要特性\n\n- 精心筛选的 300 个 Issue-Pull Request 任务\n- 覆盖 9 种编程语言的 42 个真实开源仓库\n- 每个实例使用独立的 SWE-bench Docker 沙箱环境进行多轮代理交互\n- 使用 fail-to-pass 和 pass-to-pass 测试对补丁进行 SWE-bench 兼容性评估\n\n## 评估说明\n\n- 评估前需安装 `pip install swebench==4.1.0`\n- 自动使用官方 SWE-bench Multilingual x86_64 实例镜像，并自动设置 Docker 平台为 `linux/amd64`\n- 每个实例的 Docker 镜像会自动构建或拉取\n- 每个实例最终补丁验证超时时间为 1800 秒（30 分钟）\n- 详细设置说明请参阅 [使用文档](https://evalscope.readthedocs.io/zh-cn/latest/third_party/swe_bench.html)\n- 支持本地构建镜像和远程拉取镜像两种方式\n\n## 代理模式\n\n该基准测试在每个实例专属的 SWE-bench Docker 容器内驱动一个多轮代理循环（与 mini-swe-agent 的 `swebench.yaml` 配置一致）。模型通过执行 `bash` 命令探索 `/testbed` 目录、编辑源文件，并在完成任务后打印哨兵字符串 `COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` 及其后的补丁内容来提交 `git diff` 补丁。\n\n`extra_params.action_protocol` 参数用于选择以下两种协议之一：\n- `toolcall`（默认）：采用 OpenAI 函数调用协议，仅提供一个 `bash` 工具。推荐用于支持函数调用的模型。\n- `backticks`：基于文本的备用方案，每轮期望一个 ` ```mswea_bash_command ``` ` 代码块。适用于不支持函数调用的模型。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `swe_bench_multilingual_agentic` |\n| **数据集ID** | [SWE-bench/SWE-bench_Multilingual](https://modelscope.cn/datasets/SWE-bench/SWE-bench_Multilingual/summary) |\n| **论文** | N/A |\n| **标签** | `Coding` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 300 |\n| 提示词长度（平均） | 2197.94 字符 |\n| 提示词长度（最小/最大） | 124 / 69351 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"52ece3cf\",\n \"content\": \"Support Post aggregation function pow(f1,f2) to cater for square, cube , square root.\\n### Description\\r\\n\\r\\nPlease describe the feature or change with as much detail as possible. \\r\\n\\r\\nAs of now the only supported arithmetic functions are +, -, *, ... [TRUNCATED 284 chars] ... \\r\\n\\r\\nThe proposal is to add a `pow` function which enables all the about usecase . Square of a number can be represent by pow(f1,2) , Cube can be represented as power(f1 ,3) , Squar root of a number can be represented by power(f1,0.5) ,\\r\\n\\r\\n\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"problem_statement\": \"Support Post aggregation function pow(f1,f2) to cater for square, cube , square root.\\n### Description\\r\\n\\r\\nPlease describe the feature or change with as much detail as possible. \\r\\n\\r\\nAs of now the only supported arithmetic functions are +, -, *, ... [TRUNCATED 284 chars] ... \\r\\n\\r\\nThe proposal is to add a `pow` function which enables all the about usecase . Square of a number can be represent by pow(f1,2) , Cube can be represented as power(f1 ,3) , Squar root of a number can be represented by power(f1,0.5) ,\\r\\n\\r\\n\\n\",\n \"instance_id\": \"apache__druid-13704\",\n \"base_commit\": \"51dfde02840017092486fb75be2b16566aff6a19\",\n \"patch\": \"diff --git a/docs/querying/post-aggregations.md b/docs/querying/post-aggregations.md\\nindex c75d122eb20a..935ca8fbce16 100644\\n--- a/docs/querying/post-aggregations.md\\n+++ b/docs/querying/post-aggregations.md\\n@@ -36,7 +36,7 @@ There are several ... [TRUNCATED 1110 chars] ... }\\n+ },\\n+\\n+ POW(\\\"pow\\\") {\\n+ @Override\\n+ public double compute(double lhs, double rhs)\\n+ {\\n+ return Math.pow(lhs, rhs);\\n+ }\\n };\\n \\n private static final Map LOOKUP_MAP = new HashMap<>();\\n\",\n \"PASS_TO_PASS\": [\n \"org.apache.druid.query.aggregation.post.ArithmeticPostAggregatorTest#testDiv\",\n \"org.apache.druid.query.aggregation.post.ArithmeticPostAggregatorTest#testQuotient\"\n ],\n \"FAIL_TO_PASS\": [\n \"org.apache.druid.query.aggregation.post.ArithmeticPostAggregatorTest#testPow\"\n ],\n \"test_patch\": \"diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java\\nindex a93034427539..7e1d4d112 ... [TRUNCATED 1358 chars] ... s(1.0, agg.compute(ImmutableMap.of(\\\"value\\\", 1)));\\n+ Assert.assertEquals(1.0, agg.compute(ImmutableMap.of(\\\"value\\\", -1)));\\n+ Assert.assertEquals(1.0, agg.compute(ImmutableMap.of(\\\"value\\\", .5)));\\n+ }\\n @Test\\n public void testDiv()\\n {\\n\",\n \"version\": \"13704\",\n \"repo\": \"apache/druid\",\n \"environment_setup_commit\": null,\n \"hints_text\": \"\",\n \"created_at\": \"2023-01-23 04:10:47\",\n \"docker_image\": \"swebench/sweb.eval.x86_64.apache_1776_druid-13704:latest\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `action_protocol` | `str` | `toolcall` | 代理动作协议：\"toolcall\"（主流 OpenAI 函数调用方式，与 mini-swe-agent swebench.yaml 一致）或 \"backticks\"（针对不支持函数调用的模型的文本式 mswea_bash_command 回退方案）。可选值：['toolcall', 'backticks'] |\n| `max_steps` | `int` | `250` | 每个样本的最大代理步数。 |\n| `command_timeout` | `float` | `60.0` | 每个 bash 命令的默认超时时间（秒）。 |\n| `build_docker_images` | `bool` | `True` | 为每个样本在本地构建 Docker 镜像。 |\n| `pull_remote_images_if_available` | `bool` | `True` | 在构建前尝试拉取已存在的远程 Docker 镜像。 |\n| `force_arch` | `str` | `` | 可选地强制指定镜像构建/拉取的架构。可选值：['', 'arm64', 'x86_64'] |\n| `dockerhub_username` | `str` | `swebench` | 远程 SWE-bench 镜像在 DockerHub 上的用户/组织命名空间。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_multilingual_agentic \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_multilingual_agentic'],\n dataset_args={\n 'swe_bench_multilingual_agentic': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "2b3e443c34418426ab07cb95d176a930", + "en": "# SWE-bench_Multilingual_Agentic\n\n\n## Overview\n\nSWE-bench Multilingual Agentic is the agentic-mode evaluation of SWE-bench Multilingual, a 300-task SWE-bench-style benchmark spanning 42 repositories and 9 programming languages. The model autonomously explores, edits, and submits a patch through a multi-turn agent loop inside a per-instance Docker container.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic, Multilingual)\n- **Input**: GitHub issue description (no oracle file context)\n- **Output**: Code patch (diff format) collected from `git diff` after autonomous editing\n- **Languages**: C, C++, Go, Java, JavaScript/TypeScript, PHP, Ruby, and Rust\n\n## Key Features\n\n- 300 curated Issue-Pull Request tasks\n- 42 real-world repositories across 9 programming languages\n- Multi-turn agent loop with per-instance SWE-bench Docker sandbox\n- SWE-bench-compatible patch evaluation using fail-to-pass and pass-to-pass tests\n\n## Evaluation Notes\n\n- Requires `pip install swebench==4.1.0` before evaluation\n- Uses the official SWE-bench Multilingual x86_64 instance images and sets Docker platform to `linux/amd64` automatically\n- Docker images are built/pulled automatically for each instance\n- Timeout of 1800 seconds (30 min) per instance for final patch validation\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench.html) for detailed setup instructions\n- Supports both local image building and remote image pulling\n\n## Agentic Mode\n\nThis benchmark drives a multi-turn agent loop (mirrors mini-swe-agent's\n`swebench.yaml`) inside a per-instance SWE-bench Docker container. The\nmodel issues `bash` commands to explore `/testbed`, edit source files,\nand finally submits its `git diff` patch by printing the sentinel\n`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` followed by the patch contents.\n\nThe default `swe_bench_toolcall` strategy uses OpenAI function calling with\na single `bash` tool. Models without function-calling support can select\n`swe_bench_backticks` through `NativeAgentConfig.strategy`; that strategy\nexpects one ` ```mswea_bash_command ``` ` block per turn.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `swe_bench_multilingual_agentic` |\n| **Dataset ID** | [SWE-bench/SWE-bench_Multilingual](https://modelscope.cn/datasets/SWE-bench/SWE-bench_Multilingual/summary) |\n| **Paper** | N/A |\n| **Tags** | `Coding` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 300 |\n| Prompt Length (Mean) | 2197.94 chars |\n| Prompt Length (Min/Max) | 124 / 69351 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"52ece3cf\",\n \"content\": \"Support Post aggregation function pow(f1,f2) to cater for square, cube , square root.\\n### Description\\r\\n\\r\\nPlease describe the feature or change with as much detail as possible. \\r\\n\\r\\nAs of now the only supported arithmetic functions are +, -, *, ... [TRUNCATED 284 chars] ... \\r\\n\\r\\nThe proposal is to add a `pow` function which enables all the about usecase . Square of a number can be represent by pow(f1,2) , Cube can be represented as power(f1 ,3) , Squar root of a number can be represented by power(f1,0.5) ,\\r\\n\\r\\n\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"problem_statement\": \"Support Post aggregation function pow(f1,f2) to cater for square, cube , square root.\\n### Description\\r\\n\\r\\nPlease describe the feature or change with as much detail as possible. \\r\\n\\r\\nAs of now the only supported arithmetic functions are +, -, *, ... [TRUNCATED 284 chars] ... \\r\\n\\r\\nThe proposal is to add a `pow` function which enables all the about usecase . Square of a number can be represent by pow(f1,2) , Cube can be represented as power(f1 ,3) , Squar root of a number can be represented by power(f1,0.5) ,\\r\\n\\r\\n\\n\",\n \"instance_id\": \"apache__druid-13704\",\n \"base_commit\": \"51dfde02840017092486fb75be2b16566aff6a19\",\n \"patch\": \"diff --git a/docs/querying/post-aggregations.md b/docs/querying/post-aggregations.md\\nindex c75d122eb20a..935ca8fbce16 100644\\n--- a/docs/querying/post-aggregations.md\\n+++ b/docs/querying/post-aggregations.md\\n@@ -36,7 +36,7 @@ There are several ... [TRUNCATED 1110 chars] ... }\\n+ },\\n+\\n+ POW(\\\"pow\\\") {\\n+ @Override\\n+ public double compute(double lhs, double rhs)\\n+ {\\n+ return Math.pow(lhs, rhs);\\n+ }\\n };\\n \\n private static final Map LOOKUP_MAP = new HashMap<>();\\n\",\n \"PASS_TO_PASS\": [\n \"org.apache.druid.query.aggregation.post.ArithmeticPostAggregatorTest#testDiv\",\n \"org.apache.druid.query.aggregation.post.ArithmeticPostAggregatorTest#testQuotient\"\n ],\n \"FAIL_TO_PASS\": [\n \"org.apache.druid.query.aggregation.post.ArithmeticPostAggregatorTest#testPow\"\n ],\n \"test_patch\": \"diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java\\nindex a93034427539..7e1d4d112 ... [TRUNCATED 1358 chars] ... s(1.0, agg.compute(ImmutableMap.of(\\\"value\\\", 1)));\\n+ Assert.assertEquals(1.0, agg.compute(ImmutableMap.of(\\\"value\\\", -1)));\\n+ Assert.assertEquals(1.0, agg.compute(ImmutableMap.of(\\\"value\\\", .5)));\\n+ }\\n @Test\\n public void testDiv()\\n {\\n\",\n \"version\": \"13704\",\n \"repo\": \"apache/druid\",\n \"environment_setup_commit\": null,\n \"hints_text\": \"\",\n \"created_at\": \"2023-01-23 04:10:47\",\n \"docker_image\": \"swebench/sweb.eval.x86_64.apache_1776_druid-13704:latest\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `build_docker_images` | `bool` | `True` | Build Docker images locally for each sample. |\n| `pull_remote_images_if_available` | `bool` | `True` | Attempt to pull existing remote Docker images before building. |\n| `force_arch` | `str` | `` | Optionally force a specific architecture for image build/pull. Choices: ['', 'arm64', 'x86_64'] |\n| `dockerhub_username` | `str` | `swebench` | DockerHub user/org namespace for remote SWE-bench images. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_multilingual_agentic \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"swe_bench_toolcall\",\"max_steps\":250}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_multilingual_agentic'],\n agent_config=NativeAgentConfig(\n strategy='swe_bench_toolcall',\n max_steps=250,\n ),\n dataset_args={\n 'swe_bench_multilingual_agentic': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# SWE-bench_Multilingual_Agentic\n\n\n## 概述\n\nSWE-bench Multilingual Agentic 是 SWE-bench Multilingual 的代理模式（agentic-mode）评估版本。该基准测试包含 300 个 SWE-bench 风格的任务，覆盖 42 个代码仓库和 9 种编程语言。模型在每个实例独立的 Docker 容器中，通过多轮代理循环自主探索、编辑代码并提交补丁。\n\n## 任务描述\n\n- **任务类型**：自动化软件工程 / 缺陷修复（代理式、多语言）\n- **输入**：GitHub issue 描述（不提供 oracle 文件上下文）\n- **输出**：模型自主编辑后通过 `git diff` 生成的代码补丁（diff 格式）\n- **支持语言**：C、C++、Go、Java、JavaScript/TypeScript、PHP、Ruby 和 Rust\n\n## 主要特性\n\n- 精心筛选的 300 个 Issue-Pull Request 任务\n- 覆盖 9 种编程语言的 42 个真实开源仓库\n- 每个实例使用独立的 SWE-bench Docker 沙箱进行多轮代理交互\n- 使用 fail-to-pass 和 pass-to-pass 测试，与 SWE-bench 兼容的补丁评估机制\n\n## 评估说明\n\n- 评估前需安装 `pip install swebench==4.1.0`\n- 自动使用官方 SWE-bench Multilingual x86_64 实例镜像，并自动设置 Docker 平台为 `linux/amd64`\n- 每个实例的 Docker 镜像会自动构建或拉取\n- 每个实例最终补丁验证的超时时间为 1800 秒（30 分钟）\n- 详细设置说明请参阅 [使用文档](https://evalscope.readthedocs.io/zh-cn/latest/third_party/swe_bench.html)\n- 支持本地构建镜像和远程拉取镜像两种方式\n\n## 代理模式\n\n本基准测试在每个实例的 SWE-bench Docker 容器内驱动一个多轮代理循环（与 mini-swe-agent 的 `swebench.yaml` 配置一致）。模型通过发出 `bash` 命令探索 `/testbed` 目录、编辑源文件，并在完成任务时打印哨兵字符串 `COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` 后附上补丁内容以提交最终 `git diff` 补丁。\n\n默认的 `swe_bench_toolcall` 策略使用 OpenAI 函数调用，仅提供一个 `bash` 工具。对于不支持函数调用的模型，可通过 `NativeAgentConfig.strategy` 选择 `swe_bench_backticks` 策略；该策略要求每轮输出一个 ` ```mswea_bash_command ``` ` 代码块。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `swe_bench_multilingual_agentic` |\n| **数据集ID** | [SWE-bench/SWE-bench_Multilingual](https://modelscope.cn/datasets/SWE-bench/SWE-bench_Multilingual/summary) |\n| **论文** | N/A |\n| **标签** | `Coding` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 300 |\n| 提示词长度（平均） | 2197.94 字符 |\n| 提示词长度（最小/最大） | 124 / 69351 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"52ece3cf\",\n \"content\": \"Support Post aggregation function pow(f1,f2) to cater for square, cube , square root.\\n### Description\\r\\n\\r\\nPlease describe the feature or change with as much detail as possible. \\r\\n\\r\\nAs of now the only supported arithmetic functions are +, -, *, ... [TRUNCATED 284 chars] ... \\r\\n\\r\\nThe proposal is to add a `pow` function which enables all the about usecase . Square of a number can be represent by pow(f1,2) , Cube can be represented as power(f1 ,3) , Squar root of a number can be represented by power(f1,0.5) ,\\r\\n\\r\\n\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"problem_statement\": \"Support Post aggregation function pow(f1,f2) to cater for square, cube , square root.\\n### Description\\r\\n\\r\\nPlease describe the feature or change with as much detail as possible. \\r\\n\\r\\nAs of now the only supported arithmetic functions are +, -, *, ... [TRUNCATED 284 chars] ... \\r\\n\\r\\nThe proposal is to add a `pow` function which enables all the about usecase . Square of a number can be represent by pow(f1,2) , Cube can be represented as power(f1 ,3) , Squar root of a number can be represented by power(f1,0.5) ,\\r\\n\\r\\n\\n\",\n \"instance_id\": \"apache__druid-13704\",\n \"base_commit\": \"51dfde02840017092486fb75be2b16566aff6a19\",\n \"patch\": \"diff --git a/docs/querying/post-aggregations.md b/docs/querying/post-aggregations.md\\nindex c75d122eb20a..935ca8fbce16 100644\\n--- a/docs/querying/post-aggregations.md\\n+++ b/docs/querying/post-aggregations.md\\n@@ -36,7 +36,7 @@ There are several ... [TRUNCATED 1110 chars] ... }\\n+ },\\n+\\n+ POW(\\\"pow\\\") {\\n+ @Override\\n+ public double compute(double lhs, double rhs)\\n+ {\\n+ return Math.pow(lhs, rhs);\\n+ }\\n };\\n \\n private static final Map LOOKUP_MAP = new HashMap<>();\\n\",\n \"PASS_TO_PASS\": [\n \"org.apache.druid.query.aggregation.post.ArithmeticPostAggregatorTest#testDiv\",\n \"org.apache.druid.query.aggregation.post.ArithmeticPostAggregatorTest#testQuotient\"\n ],\n \"FAIL_TO_PASS\": [\n \"org.apache.druid.query.aggregation.post.ArithmeticPostAggregatorTest#testPow\"\n ],\n \"test_patch\": \"diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java\\nindex a93034427539..7e1d4d112 ... [TRUNCATED 1358 chars] ... s(1.0, agg.compute(ImmutableMap.of(\\\"value\\\", 1)));\\n+ Assert.assertEquals(1.0, agg.compute(ImmutableMap.of(\\\"value\\\", -1)));\\n+ Assert.assertEquals(1.0, agg.compute(ImmutableMap.of(\\\"value\\\", .5)));\\n+ }\\n @Test\\n public void testDiv()\\n {\\n\",\n \"version\": \"13704\",\n \"repo\": \"apache/druid\",\n \"environment_setup_commit\": null,\n \"hints_text\": \"\",\n \"created_at\": \"2023-01-23 04:10:47\",\n \"docker_image\": \"swebench/sweb.eval.x86_64.apache_1776_druid-13704:latest\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `build_docker_images` | `bool` | `True` | 为每个样本在本地构建 Docker 镜像。 |\n| `pull_remote_images_if_available` | `bool` | `True` | 在构建前尝试拉取已存在的远程 Docker 镜像。 |\n| `force_arch` | `str` | `` | 可选地强制指定镜像构建/拉取的目标架构。可选值：['', 'arm64', 'x86_64'] |\n| `dockerhub_username` | `str` | `swebench` | 远程 SWE-bench 镜像在 DockerHub 上的用户/组织命名空间。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_multilingual_agentic \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"swe_bench_toolcall\",\"max_steps\":250}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_multilingual_agentic'],\n agent_config=NativeAgentConfig(\n strategy='swe_bench_toolcall',\n max_steps=250,\n ),\n dataset_args={\n 'swe_bench_multilingual_agentic': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "77c56224cf2bdb123d23e1461c659dd9", "needs_translation": false }, - "updated_at": "2026-07-02T19:24:57.846459", - "translation_updated_at": "2026-07-02T19:26:43" -} \ No newline at end of file + "updated_at": "2026-07-10T20:24:34.534425", + "translation_updated_at": "2026-07-10T20:24:42" +} diff --git a/evalscope/evalscope/benchmarks/_meta/swe_bench_pro.json b/evalscope/evalscope/benchmarks/_meta/swe_bench_pro.json index e21916a..918eaa7 100644 --- a/evalscope/evalscope/benchmarks/_meta/swe_bench_pro.json +++ b/evalscope/evalscope/benchmarks/_meta/swe_bench_pro.json @@ -31,25 +31,6 @@ "description": "DockerHub user/org hosting the sweap-images repository.", "value": "jefzda" }, - "action_protocol": { - "type": "str", - "description": "Agent action protocol: \"toolcall\" (function-calling) or \"backticks\" (text-based fallback for models without function-calling support).", - "value": "toolcall", - "choices": [ - "toolcall", - "backticks" - ] - }, - "max_steps": { - "type": "int", - "description": "Maximum number of agent steps per sample.", - "value": 250 - }, - "command_timeout": { - "type": "float", - "description": "Default per-bash-command timeout in seconds.", - "value": 60.0 - }, "eval_timeout": { "type": "int", "description": "Per-instance evaluation timeout in seconds.", @@ -57,6 +38,10 @@ } }, "sandbox_config": {}, + "agent_config": { + "strategy": "swe_bench_toolcall", + "max_steps": 250 + }, "category": "agent" }, "statistics": { @@ -137,11 +122,11 @@ "truncated": false }, "readme": { - "en": "# SWE-bench_Pro\n\n\n## Overview\n\nSWE-bench_Pro is a challenging benchmark from Scale AI evaluating LLMs/Agents on long-horizon software engineering tasks across multiple programming languages. Given a codebase and an issue, the model must autonomously explore the repository, edit source files, and submit a patch through a multi-turn agent loop inside a per-instance Docker container.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic)\n- **Input**: GitHub issue description\n- **Output**: Code patch (diff format) collected after autonomous editing\n- **Languages**: Multiple (`repo_language` field; e.g. JavaScript/TypeScript, Python, Go)\n\n## Key Features\n\n- Multi-turn agent loop with per-instance DockerHub image (`jefzda/sweap-images:{tag}`)\n- Sentinel-based patch submission protocol (`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`)\n- Container-side evaluation: `git apply` patch, run instance's `run_script.sh`, parse with `parser.py`, then check `(fail_to_pass | pass_to_pass) ⊆ PASSED`\n- Supports both `toolcall` (function-calling) and `backticks` (text-based) action protocols\n\n## Evaluation Notes\n\n- Requires `pip install evalscope[sandbox]` (provides Docker SDK via ms-enclave)\n- Requires the `scaleapi/SWE-bench_Pro-os` repository for per-instance run scripts and Dockerfiles. By default this is auto-cloned to `~/.cache/evalscope/swe_bench_pro/SWE-bench_Pro-os` and pinned to commit `ca10a60`. To use an existing clone, set `extra_params.swe_bench_pro_repo_path`.\n- Both the agent loop and the per-instance evaluation share a single sandbox configuration via `TaskConfig.sandbox.default_config` (passed straight to ms_enclave `DockerSandboxConfig`). Set `memory_limit` / `cpu_limit` there to avoid OOM-Killed test runs (e.g. NodeBB); `platform` defaults to `linux/amd64` so amd64-only sweap-images work on Apple Silicon out of the box.\n\nSee the [user guide](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench_pro.html) for setup, parameters, and troubleshooting.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `swe_bench_pro` |\n| **Dataset ID** | [ScaleAI/SWE-bench_Pro](https://modelscope.cn/datasets/ScaleAI/SWE-bench_Pro/summary) |\n| **Paper** | N/A |\n| **Tags** | `Coding` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 731 |\n| Prompt Length (Mean) | 1297.47 chars |\n| Prompt Length (Min/Max) | 419 / 8036 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"3874fd29\",\n \"content\": \"\\\"**Title: Email Validation Status Not Handled Correctly in ACP and Confirmation Logic**\\\\n\\\\n**Description:**\\\\n\\\\nThe Admin Control Panel (ACP) does not accurately reflect the email validation status of users. Also, validation and confirmation p ... [TRUNCATED 846 chars] ... mail validation.\\\\n\\\\nThe email status was unclear or incorrect in ACP.\\\\n\\\\n\\\\\\\"Validate\\\\\\\" and \\\\\\\"Send validation email\\\\\\\" actions failed when the expected data was missing.\\\\n\\\\n**Labels:**\\\\n\\\\nbug, back-end, authentication, ui/ux, email-confirmation\\\"\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"instance_id\": \"instance_NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5-vnan\",\n \"repo\": \"NodeBB/NodeBB\",\n \"base_commit\": \"1e137b07052bc3ea0da44ed201702c94055b8ad2\",\n \"problem_statement\": \"\\\"**Title: Email Validation Status Not Handled Correctly in ACP and Confirmation Logic**\\\\n\\\\n**Description:**\\\\n\\\\nThe Admin Control Panel (ACP) does not accurately reflect the email validation status of users. Also, validation and confirmation p ... [TRUNCATED 846 chars] ... mail validation.\\\\n\\\\nThe email status was unclear or incorrect in ACP.\\\\n\\\\n\\\\\\\"Validate\\\\\\\" and \\\\\\\"Send validation email\\\\\\\" actions failed when the expected data was missing.\\\\n\\\\n**Labels:**\\\\n\\\\nbug, back-end, authentication, ui/ux, email-confirmation\\\"\",\n \"patch\": \"diff --git a/public/language/en-GB/admin/manage/users.json b/public/language/en-GB/admin/manage/users.json\\nindex 6b668a31ef8e..9486295bc3ef 100644\\n--- a/public/language/en-GB/admin/manage/users.json\\n+++ b/public/language/en-GB/admin/manage/us ... [TRUNCATED 12136 chars] ... class=\\\"notvalidated fa fa-check text-muted\\\" title=\\\"not validated\\\">\\n+\\t\\t\\t\\t\\t\\t\\t\\t\\n \\t\\t\\t\\t\\t\\t\\t\\t[[admin/manage/users:users.no-email]]\\n \\t\\t\\t\\t\\t\\t\\t\\t{{{ end }}}\\n \\t\\t\\t\\t\\t\\t\\t\\n\",\n \"test_patch\": \"diff --git a/test/database/keys.js b/test/database/keys.js\\nindex 3941edb65a93..fde4bbc442cf 100644\\n--- a/test/database/keys.js\\n+++ b/test/database/keys.js\\n@@ -35,6 +35,17 @@ describe('Key methods', () => {\\n \\t\\t});\\n \\t});\\n \\n+\\tit('should return m ... [TRUNCATED 952 chars] ... {uid}`, 1000);\\n+\\t\\t\\tconst code = await db.get(`confirm:byUid:${uid}`);\\n+\\t\\t\\tawait db.setObjectField(`confirm:${code}`, 'expires', Date.now() + 1000);\\n \\t\\t\\tconst ok = await user.email.canSendValidation(uid, email);\\n-\\n \\t\\t\\tassert(ok);\\n \\t\\t});\\n \\t});\\n\",\n \"fail_to_pass\": \"[\\\"test/database.js | Test database test/database/keys.js::Key methods should return multiple keys and null if key doesn't exist\\\", 'test/database.js | Test database test/database/keys.js::Key methods should return empty array if keys is empty array or falsy', 'test/user/emails.js | email confirmation (library methods) canSendValidation should return true if it has been long enough to re-send confirmation']\",\n \"pass_to_pass\": \"[\\\"test/database.js | Test database should work\\\", \\\"test/database.js | Test database info should return info about database\\\", \\\"test/database.js | Test database info should not error and return info if client is falsy\\\", \\\"test/database.js | Test ... [TRUNCATED 45280 chars] ... pending or set\\\", \\\"test/user/emails.js | email confirmation (v3 api) should confirm their email (using the pending validation)\\\", \\\"test/user/emails.js | email confirmation (v3 api) should still confirm the email (as email is set in user hash)\\\"]\",\n \"before_repo_set_cmd\": \"git reset --hard 1e137b07052bc3ea0da44ed201702c94055b8ad2\\ngit clean -fd \\ngit checkout 1e137b07052bc3ea0da44ed201702c94055b8ad2 \\ngit checkout 04998908ba6721d64eba79ae3b65a351dcfbc5b5 -- test/database/keys.js test/user/emails.js\",\n \"selected_test_files_to_run\": \"[\\\"test/database.js\\\", \\\"test/database/keys.js\\\", \\\"test/user/emails.js\\\"]\",\n \"repo_language\": \"js\",\n \"requirements\": \"\\\"- The loadUserInfo(callerUid, uids) function should include logic to retrieve and attach `email:pending` and `email:expired` flags to each user object. These flags must be derived by resolving `confirm:byUid:` keys via the new `getConfi ... [TRUNCATED 3170 chars] ... rval check must compare the stored TTL timestamp if available (or, if TTL is unavailable, use the current time as the baseline) plus the configured interval against the max confirmation period, ensuring the system prevents excessive resends.\\\"\",\n \"interface\": \"\\\"Type: Method\\\\n\\\\nName: db.mget\\\\n\\\\nPath: src/database/mongo/main.js, src/database/postgres/main.js, src/database/redis/main.js\\\\n\\\\nInput: keys: string[] (An array of database keys to retrieve.)\\\\n\\\\nOutput: Promise<(string | null)[]> (A promise t ... [TRUNCATED 487 chars] ... to the email address string, or `null` if no suitable email is found.)\\\\n\\\\nDescription: A utility function that retrieves the most appropriate email address for an administrative action like \\\\\\\"force validate\\\\\\\" or \\\\\\\"resend validation email\\\\\\\".\\\"\",\n \"issue_specificity\": \"[\\\"major_bug\\\",\\\"data_bug\\\",\\\"ui_ux_bug\\\"]\",\n \"issue_categories\": \"[\\\"back_end_knowledge\\\",\\\"database_knowledge\\\",\\\"authentication_authorization_knowledge\\\",\\\"ui_ux_knowledge\\\"]\",\n \"dockerhub_tag\": \"nodebb.nodebb-NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5\",\n \"docker_image\": \"jefzda/sweap-images:nodebb.nodebb-NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `swe_bench_pro_repo_path` | `str` | `` | Local path to a clone of scaleapi/SWE-bench_Pro-os. If empty, auto-cloned to ~/.cache/evalscope/swe_bench_pro/SWE-bench_Pro-os and pinned to commit ca10a60. |\n| `dockerhub_username` | `str` | `jefzda` | DockerHub user/org hosting the sweap-images repository. |\n| `action_protocol` | `str` | `toolcall` | Agent action protocol: \"toolcall\" (function-calling) or \"backticks\" (text-based fallback for models without function-calling support). Choices: ['toolcall', 'backticks'] |\n| `max_steps` | `int` | `250` | Maximum number of agent steps per sample. |\n| `command_timeout` | `float` | `60.0` | Default per-bash-command timeout in seconds. |\n| `eval_timeout` | `int` | `3600` | Per-instance evaluation timeout in seconds. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_pro \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_pro'],\n dataset_args={\n 'swe_bench_pro': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# SWE-bench_Pro\n\n\n## 概述\n\nSWE-bench_Pro 是由 Scale AI 提供的一项具有挑战性的基准测试，用于评估大语言模型（LLM）或智能体在多种编程语言环境下执行长周期软件工程任务的能力。给定一个代码库和一个问题描述，模型必须在每个实例专属的 Docker 容器内，通过多轮智能体交互循环，自主探索代码仓库、编辑源文件，并最终提交补丁。\n\n## 任务描述\n\n- **任务类型**：自动化软件工程 / 缺陷修复（智能体驱动）\n- **输入**：GitHub issue 描述\n- **输出**：自主编辑后收集的代码补丁（diff 格式）\n- **支持语言**：多种（由 `repo_language` 字段指定；例如 JavaScript/TypeScript、Python、Go）\n\n## 核心特性\n\n- 基于每实例 DockerHub 镜像（`jefzda/sweap-images:{tag}`）的多轮智能体交互循环\n- 基于哨兵（sentinel）的补丁提交协议（`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`）\n- 容器端评估流程：应用 `git apply` 打补丁，运行实例对应的 `run_script.sh` 脚本，通过 `parser.py` 解析结果，并验证 `(fail_to_pass | pass_to_pass) ⊆ PASSED`\n- 同时支持 `toolcall`（函数调用）和 `backticks`（基于文本）两种动作协议\n\n## 评估说明\n\n- 需要安装 `pip install evalscope[sandbox]`（通过 ms-enclave 提供 Docker SDK）\n- 需要 `scaleapi/SWE-bench_Pro-os` 仓库以获取每实例的运行脚本和 Dockerfile。默认情况下，该仓库会自动克隆至 `~/.cache/evalscope/swe_bench_pro/SWE-bench_Pro-os` 并固定到 commit `ca10a60`。若要使用已有克隆，请设置 `extra_params.swe_bench_pro_repo_path`。\n- 智能体循环与每实例评估共享同一个沙箱配置（通过 `TaskConfig.sandbox.default_config` 直接传递给 ms_enclave 的 `DockerSandboxConfig`）。建议在此处设置 `memory_limit` / `cpu_limit`，以避免因内存不足导致测试被终止（例如 NodeBB 实例）；`platform` 默认为 `linux/amd64`，因此仅支持 amd64 的 sweap-images 可在 Apple Silicon 设备上开箱即用。\n\n有关环境设置、参数配置及故障排查，请参阅[用户指南](https://evalscope.readthedocs.io/zh-cn/latest/third_party/swe_bench_pro.html)。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `swe_bench_pro` |\n| **数据集ID** | [ScaleAI/SWE-bench_Pro](https://modelscope.cn/datasets/ScaleAI/SWE-bench_Pro/summary) |\n| **论文** | N/A |\n| **标签** | `Coding` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 731 |\n| 提示词长度（平均） | 1297.47 字符 |\n| 提示词长度（最小/最大） | 419 / 8036 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"3874fd29\",\n \"content\": \"\\\"**Title: Email Validation Status Not Handled Correctly in ACP and Confirmation Logic**\\\\n\\\\n**Description:**\\\\n\\\\nThe Admin Control Panel (ACP) does not accurately reflect the email validation status of users. Also, validation and confirmation p ... [TRUNCATED 846 chars] ... mail validation.\\\\n\\\\nThe email status was unclear or incorrect in ACP.\\\\n\\\\n\\\\\\\"Validate\\\\\\\" and \\\\\\\"Send validation email\\\\\\\" actions failed when the expected data was missing.\\\\n\\\\n**Labels:**\\\\n\\\\nbug, back-end, authentication, ui/ux, email-confirmation\\\"\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"instance_id\": \"instance_NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5-vnan\",\n \"repo\": \"NodeBB/NodeBB\",\n \"base_commit\": \"1e137b07052bc3ea0da44ed201702c94055b8ad2\",\n \"problem_statement\": \"\\\"**Title: Email Validation Status Not Handled Correctly in ACP and Confirmation Logic**\\\\n\\\\n**Description:**\\\\n\\\\nThe Admin Control Panel (ACP) does not accurately reflect the email validation status of users. Also, validation and confirmation p ... [TRUNCATED 846 chars] ... mail validation.\\\\n\\\\nThe email status was unclear or incorrect in ACP.\\\\n\\\\n\\\\\\\"Validate\\\\\\\" and \\\\\\\"Send validation email\\\\\\\" actions failed when the expected data was missing.\\\\n\\\\n**Labels:**\\\\n\\\\nbug, back-end, authentication, ui/ux, email-confirmation\\\"\",\n \"patch\": \"diff --git a/public/language/en-GB/admin/manage/users.json b/public/language/en-GB/admin/manage/users.json\\nindex 6b668a31ef8e..9486295bc3ef 100644\\n--- a/public/language/en-GB/admin/manage/users.json\\n+++ b/public/language/en-GB/admin/manage/us ... [TRUNCATED 12136 chars] ... class=\\\"notvalidated fa fa-check text-muted\\\" title=\\\"not validated\\\">\\n+\\t\\t\\t\\t\\t\\t\\t\\t\\n \\t\\t\\t\\t\\t\\t\\t\\t[[admin/manage/users:users.no-email]]\\n \\t\\t\\t\\t\\t\\t\\t\\t{{{ end }}}\\n \\t\\t\\t\\t\\t\\t\\t\\n\",\n \"test_patch\": \"diff --git a/test/database/keys.js b/test/database/keys.js\\nindex 3941edb65a93..fde4bbc442cf 100644\\n--- a/test/database/keys.js\\n+++ b/test/database/keys.js\\n@@ -35,6 +35,17 @@ describe('Key methods', () => {\\n \\t\\t});\\n \\t});\\n \\n+\\tit('should return m ... [TRUNCATED 952 chars] ... {uid}`, 1000);\\n+\\t\\t\\tconst code = await db.get(`confirm:byUid:${uid}`);\\n+\\t\\t\\tawait db.setObjectField(`confirm:${code}`, 'expires', Date.now() + 1000);\\n \\t\\t\\tconst ok = await user.email.canSendValidation(uid, email);\\n-\\n \\t\\t\\tassert(ok);\\n \\t\\t});\\n \\t});\\n\",\n \"fail_to_pass\": \"[\\\"test/database.js | Test database test/database/keys.js::Key methods should return multiple keys and null if key doesn't exist\\\", 'test/database.js | Test database test/database/keys.js::Key methods should return empty array if keys is empty array or falsy', 'test/user/emails.js | email confirmation (library methods) canSendValidation should return true if it has been long enough to re-send confirmation']\",\n \"pass_to_pass\": \"[\\\"test/database.js | Test database should work\\\", \\\"test/database.js | Test database info should return info about database\\\", \\\"test/database.js | Test database info should not error and return info if client is falsy\\\", \\\"test/database.js | Test ... [TRUNCATED 45280 chars] ... pending or set\\\", \\\"test/user/emails.js | email confirmation (v3 api) should confirm their email (using the pending validation)\\\", \\\"test/user/emails.js | email confirmation (v3 api) should still confirm the email (as email is set in user hash)\\\"]\",\n \"before_repo_set_cmd\": \"git reset --hard 1e137b07052bc3ea0da44ed201702c94055b8ad2\\ngit clean -fd \\ngit checkout 1e137b07052bc3ea0da44ed201702c94055b8ad2 \\ngit checkout 04998908ba6721d64eba79ae3b65a351dcfbc5b5 -- test/database/keys.js test/user/emails.js\",\n \"selected_test_files_to_run\": \"[\\\"test/database.js\\\", \\\"test/database/keys.js\\\", \\\"test/user/emails.js\\\"]\",\n \"repo_language\": \"js\",\n \"requirements\": \"\\\"- The loadUserInfo(callerUid, uids) function should include logic to retrieve and attach `email:pending` and `email:expired` flags to each user object. These flags must be derived by resolving `confirm:byUid:` keys via the new `getConfi ... [TRUNCATED 3170 chars] ... rval check must compare the stored TTL timestamp if available (or, if TTL is unavailable, use the current time as the baseline) plus the configured interval against the max confirmation period, ensuring the system prevents excessive resends.\\\"\",\n \"interface\": \"\\\"Type: Method\\\\n\\\\nName: db.mget\\\\n\\\\nPath: src/database/mongo/main.js, src/database/postgres/main.js, src/database/redis/main.js\\\\n\\\\nInput: keys: string[] (An array of database keys to retrieve.)\\\\n\\\\nOutput: Promise<(string | null)[]> (A promise t ... [TRUNCATED 487 chars] ... to the email address string, or `null` if no suitable email is found.)\\\\n\\\\nDescription: A utility function that retrieves the most appropriate email address for an administrative action like \\\\\\\"force validate\\\\\\\" or \\\\\\\"resend validation email\\\\\\\".\\\"\",\n \"issue_specificity\": \"[\\\"major_bug\\\",\\\"data_bug\\\",\\\"ui_ux_bug\\\"]\",\n \"issue_categories\": \"[\\\"back_end_knowledge\\\",\\\"database_knowledge\\\",\\\"authentication_authorization_knowledge\\\",\\\"ui_ux_knowledge\\\"]\",\n \"dockerhub_tag\": \"nodebb.nodebb-NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5\",\n \"docker_image\": \"jefzda/sweap-images:nodebb.nodebb-NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `swe_bench_pro_repo_path` | `str` | `` | 指向 `scaleapi/SWE-bench_Pro-os` 本地克隆路径。若为空，则自动克隆至 `~/.cache/evalscope/swe_bench_pro/SWE-bench_Pro-os` 并固定到 commit `ca10a60`。 |\n| `dockerhub_username` | `str` | `jefzda` | 托管 sweap-images 仓库的 DockerHub 用户或组织名。 |\n| `action_protocol` | `str` | `toolcall` | 智能体动作协议：\"toolcall\"（函数调用）或 \"backticks\"（针对不支持函数调用模型的文本回退方案）。可选值：['toolcall', 'backticks'] |\n| `max_steps` | `int` | `250` | 每个样本允许的最大智能体步数。 |\n| `command_timeout` | `float` | `60.0` | 每条 bash 命令的默认超时时间（秒）。 |\n| `eval_timeout` | `int` | `3600` | 每实例评估的超时时间（秒）。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_pro \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_pro'],\n dataset_args={\n 'swe_bench_pro': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "40d924022cb9c4b94ecc6976a770ef10", + "en": "# SWE-bench_Pro\n\n\n## Overview\n\nSWE-bench_Pro is a challenging benchmark from Scale AI evaluating LLMs/Agents on long-horizon software engineering tasks across multiple programming languages. Given a codebase and an issue, the model must autonomously explore the repository, edit source files, and submit a patch through a multi-turn agent loop inside a per-instance Docker container.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic)\n- **Input**: GitHub issue description\n- **Output**: Code patch (diff format) collected after autonomous editing\n- **Languages**: Multiple (`repo_language` field; e.g. JavaScript/TypeScript, Python, Go)\n\n## Key Features\n\n- Multi-turn agent loop with per-instance DockerHub image (`jefzda/sweap-images:{tag}`)\n- Sentinel-based patch submission protocol (`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`)\n- Container-side evaluation: `git apply` patch, run instance's `run_script.sh`, parse with `parser.py`, then check `(fail_to_pass | pass_to_pass) ⊆ PASSED`\n- Supports both `toolcall` (function-calling) and `backticks` (text-based) action protocols\n\n## Evaluation Notes\n\n- Requires `pip install evalscope[sandbox]` (provides Docker SDK via ms-enclave)\n- Requires the `scaleapi/SWE-bench_Pro-os` repository for per-instance run scripts and Dockerfiles. By default this is auto-cloned to `~/.cache/evalscope/swe_bench_pro/SWE-bench_Pro-os` and pinned to commit `ca10a60`. To use an existing clone, set `extra_params.swe_bench_pro_repo_path`.\n- Both the agent loop and the per-instance evaluation share a single sandbox configuration via `TaskConfig.sandbox.default_config` (passed straight to ms_enclave `DockerSandboxConfig`). Set `memory_limit` / `cpu_limit` there to avoid OOM-Killed test runs (e.g. NodeBB); `platform` defaults to `linux/amd64` so amd64-only sweap-images work on Apple Silicon out of the box.\n\nSee the [user guide](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench_pro.html) for setup, parameters, and troubleshooting.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `swe_bench_pro` |\n| **Dataset ID** | [ScaleAI/SWE-bench_Pro](https://modelscope.cn/datasets/ScaleAI/SWE-bench_Pro/summary) |\n| **Paper** | N/A |\n| **Tags** | `Coding` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 731 |\n| Prompt Length (Mean) | 1297.47 chars |\n| Prompt Length (Min/Max) | 419 / 8036 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"3874fd29\",\n \"content\": \"\\\"**Title: Email Validation Status Not Handled Correctly in ACP and Confirmation Logic**\\\\n\\\\n**Description:**\\\\n\\\\nThe Admin Control Panel (ACP) does not accurately reflect the email validation status of users. Also, validation and confirmation p ... [TRUNCATED 846 chars] ... mail validation.\\\\n\\\\nThe email status was unclear or incorrect in ACP.\\\\n\\\\n\\\\\\\"Validate\\\\\\\" and \\\\\\\"Send validation email\\\\\\\" actions failed when the expected data was missing.\\\\n\\\\n**Labels:**\\\\n\\\\nbug, back-end, authentication, ui/ux, email-confirmation\\\"\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"instance_id\": \"instance_NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5-vnan\",\n \"repo\": \"NodeBB/NodeBB\",\n \"base_commit\": \"1e137b07052bc3ea0da44ed201702c94055b8ad2\",\n \"problem_statement\": \"\\\"**Title: Email Validation Status Not Handled Correctly in ACP and Confirmation Logic**\\\\n\\\\n**Description:**\\\\n\\\\nThe Admin Control Panel (ACP) does not accurately reflect the email validation status of users. Also, validation and confirmation p ... [TRUNCATED 846 chars] ... mail validation.\\\\n\\\\nThe email status was unclear or incorrect in ACP.\\\\n\\\\n\\\\\\\"Validate\\\\\\\" and \\\\\\\"Send validation email\\\\\\\" actions failed when the expected data was missing.\\\\n\\\\n**Labels:**\\\\n\\\\nbug, back-end, authentication, ui/ux, email-confirmation\\\"\",\n \"patch\": \"diff --git a/public/language/en-GB/admin/manage/users.json b/public/language/en-GB/admin/manage/users.json\\nindex 6b668a31ef8e..9486295bc3ef 100644\\n--- a/public/language/en-GB/admin/manage/users.json\\n+++ b/public/language/en-GB/admin/manage/us ... [TRUNCATED 12136 chars] ... class=\\\"notvalidated fa fa-check text-muted\\\" title=\\\"not validated\\\">\\n+\\t\\t\\t\\t\\t\\t\\t\\t\\n \\t\\t\\t\\t\\t\\t\\t\\t[[admin/manage/users:users.no-email]]\\n \\t\\t\\t\\t\\t\\t\\t\\t{{{ end }}}\\n \\t\\t\\t\\t\\t\\t\\t\\n\",\n \"test_patch\": \"diff --git a/test/database/keys.js b/test/database/keys.js\\nindex 3941edb65a93..fde4bbc442cf 100644\\n--- a/test/database/keys.js\\n+++ b/test/database/keys.js\\n@@ -35,6 +35,17 @@ describe('Key methods', () => {\\n \\t\\t});\\n \\t});\\n \\n+\\tit('should return m ... [TRUNCATED 952 chars] ... {uid}`, 1000);\\n+\\t\\t\\tconst code = await db.get(`confirm:byUid:${uid}`);\\n+\\t\\t\\tawait db.setObjectField(`confirm:${code}`, 'expires', Date.now() + 1000);\\n \\t\\t\\tconst ok = await user.email.canSendValidation(uid, email);\\n-\\n \\t\\t\\tassert(ok);\\n \\t\\t});\\n \\t});\\n\",\n \"fail_to_pass\": \"[\\\"test/database.js | Test database test/database/keys.js::Key methods should return multiple keys and null if key doesn't exist\\\", 'test/database.js | Test database test/database/keys.js::Key methods should return empty array if keys is empty array or falsy', 'test/user/emails.js | email confirmation (library methods) canSendValidation should return true if it has been long enough to re-send confirmation']\",\n \"pass_to_pass\": \"[\\\"test/database.js | Test database should work\\\", \\\"test/database.js | Test database info should return info about database\\\", \\\"test/database.js | Test database info should not error and return info if client is falsy\\\", \\\"test/database.js | Test ... [TRUNCATED 45280 chars] ... pending or set\\\", \\\"test/user/emails.js | email confirmation (v3 api) should confirm their email (using the pending validation)\\\", \\\"test/user/emails.js | email confirmation (v3 api) should still confirm the email (as email is set in user hash)\\\"]\",\n \"before_repo_set_cmd\": \"git reset --hard 1e137b07052bc3ea0da44ed201702c94055b8ad2\\ngit clean -fd \\ngit checkout 1e137b07052bc3ea0da44ed201702c94055b8ad2 \\ngit checkout 04998908ba6721d64eba79ae3b65a351dcfbc5b5 -- test/database/keys.js test/user/emails.js\",\n \"selected_test_files_to_run\": \"[\\\"test/database.js\\\", \\\"test/database/keys.js\\\", \\\"test/user/emails.js\\\"]\",\n \"repo_language\": \"js\",\n \"requirements\": \"\\\"- The loadUserInfo(callerUid, uids) function should include logic to retrieve and attach `email:pending` and `email:expired` flags to each user object. These flags must be derived by resolving `confirm:byUid:` keys via the new `getConfi ... [TRUNCATED 3170 chars] ... rval check must compare the stored TTL timestamp if available (or, if TTL is unavailable, use the current time as the baseline) plus the configured interval against the max confirmation period, ensuring the system prevents excessive resends.\\\"\",\n \"interface\": \"\\\"Type: Method\\\\n\\\\nName: db.mget\\\\n\\\\nPath: src/database/mongo/main.js, src/database/postgres/main.js, src/database/redis/main.js\\\\n\\\\nInput: keys: string[] (An array of database keys to retrieve.)\\\\n\\\\nOutput: Promise<(string | null)[]> (A promise t ... [TRUNCATED 487 chars] ... to the email address string, or `null` if no suitable email is found.)\\\\n\\\\nDescription: A utility function that retrieves the most appropriate email address for an administrative action like \\\\\\\"force validate\\\\\\\" or \\\\\\\"resend validation email\\\\\\\".\\\"\",\n \"issue_specificity\": \"[\\\"major_bug\\\",\\\"data_bug\\\",\\\"ui_ux_bug\\\"]\",\n \"issue_categories\": \"[\\\"back_end_knowledge\\\",\\\"database_knowledge\\\",\\\"authentication_authorization_knowledge\\\",\\\"ui_ux_knowledge\\\"]\",\n \"dockerhub_tag\": \"nodebb.nodebb-NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5\",\n \"docker_image\": \"jefzda/sweap-images:nodebb.nodebb-NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `swe_bench_pro_repo_path` | `str` | `` | Local path to a clone of scaleapi/SWE-bench_Pro-os. If empty, auto-cloned to ~/.cache/evalscope/swe_bench_pro/SWE-bench_Pro-os and pinned to commit ca10a60. |\n| `dockerhub_username` | `str` | `jefzda` | DockerHub user/org hosting the sweap-images repository. |\n| `eval_timeout` | `int` | `3600` | Per-instance evaluation timeout in seconds. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_pro \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"swe_bench_toolcall\",\"max_steps\":250}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_pro'],\n agent_config=NativeAgentConfig(\n strategy='swe_bench_toolcall',\n max_steps=250,\n ),\n dataset_args={\n 'swe_bench_pro': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# SWE-bench_Pro\n\n\n## 概述\n\nSWE-bench_Pro 是由 Scale AI 提供的一项具有挑战性的基准测试，用于评估大语言模型（LLM）或智能体在多种编程语言环境下执行长周期软件工程任务的能力。给定一个代码库和一个问题描述，模型必须在每个实例专属的 Docker 容器内，通过多轮智能体交互循环，自主探索代码仓库、编辑源文件，并最终提交补丁。\n\n## 任务描述\n\n- **任务类型**：自动化软件工程 / 缺陷修复（智能体驱动）\n- **输入**：GitHub issue 描述\n- **输出**：自主编辑后生成的代码补丁（diff 格式）\n- **支持语言**：多种（由 `repo_language` 字段指定；例如 JavaScript/TypeScript、Python、Go）\n\n## 核心特性\n\n- 基于每实例 DockerHub 镜像（`jefzda/sweap-images:{tag}`）的多轮智能体循环\n- 基于哨兵（sentinel）的补丁提交协议（`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`）\n- 容器内评估流程：应用 `git apply` 打补丁，运行实例对应的 `run_script.sh` 脚本，通过 `parser.py` 解析结果，并验证 `(fail_to_pass | pass_to_pass) ⊆ PASSED`\n- 同时支持 `toolcall`（函数调用）和 `backticks`（文本动作）两种动作协议\n\n## 评估说明\n\n- 需要安装 `pip install evalscope[sandbox]`（通过 ms-enclave 提供 Docker SDK）\n- 需要 `scaleapi/SWE-bench_Pro-os` 仓库以获取每实例的运行脚本和 Dockerfile。默认情况下，该仓库会自动克隆至 `~/.cache/evalscope/swe_bench_pro/SWE-bench_Pro-os` 并固定到 commit `ca10a60`。若要使用已有克隆，请设置 `extra_params.swe_bench_pro_repo_path`。\n- 智能体循环与每实例评估共享同一个沙箱配置（通过 `TaskConfig.sandbox.default_config` 直接传入 ms_enclave 的 `DockerSandboxConfig`）。建议在此处设置 `memory_limit` / `cpu_limit` 以避免因内存不足导致测试被终止（例如 NodeBB 实例）；`platform` 默认为 `linux/amd64`，因此仅支持 amd64 架构的 sweap-images 可在 Apple Silicon 设备上开箱即用。\n\n更多设置、参数及故障排查信息，请参阅 [用户指南](https://evalscope.readthedocs.io/zh-cn/latest/third_party/swe_bench_pro.html)。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `swe_bench_pro` |\n| **数据集 ID** | [ScaleAI/SWE-bench_Pro](https://modelscope.cn/datasets/ScaleAI/SWE-bench_Pro/summary) |\n| **论文** | N/A |\n| **标签** | `Coding` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 731 |\n| 提示词长度（平均） | 1297.47 字符 |\n| 提示词长度（最小/最大） | 419 / 8036 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"3874fd29\",\n \"content\": \"\\\"**Title: Email Validation Status Not Handled Correctly in ACP and Confirmation Logic**\\\\n\\\\n**Description:**\\\\n\\\\nThe Admin Control Panel (ACP) does not accurately reflect the email validation status of users. Also, validation and confirmation p ... [TRUNCATED 846 chars] ... mail validation.\\\\n\\\\nThe email status was unclear or incorrect in ACP.\\\\n\\\\n\\\\\\\"Validate\\\\\\\" and \\\\\\\"Send validation email\\\\\\\" actions failed when the expected data was missing.\\\\n\\\\n**Labels:**\\\\n\\\\nbug, back-end, authentication, ui/ux, email-confirmation\\\"\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"instance_id\": \"instance_NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5-vnan\",\n \"repo\": \"NodeBB/NodeBB\",\n \"base_commit\": \"1e137b07052bc3ea0da44ed201702c94055b8ad2\",\n \"problem_statement\": \"\\\"**Title: Email Validation Status Not Handled Correctly in ACP and Confirmation Logic**\\\\n\\\\n**Description:**\\\\n\\\\nThe Admin Control Panel (ACP) does not accurately reflect the email validation status of users. Also, validation and confirmation p ... [TRUNCATED 846 chars] ... mail validation.\\\\n\\\\nThe email status was unclear or incorrect in ACP.\\\\n\\\\n\\\\\\\"Validate\\\\\\\" and \\\\\\\"Send validation email\\\\\\\" actions failed when the expected data was missing.\\\\n\\\\n**Labels:**\\\\n\\\\nbug, back-end, authentication, ui/ux, email-confirmation\\\"\",\n \"patch\": \"diff --git a/public/language/en-GB/admin/manage/users.json b/public/language/en-GB/admin/manage/users.json\\nindex 6b668a31ef8e..9486295bc3ef 100644\\n--- a/public/language/en-GB/admin/manage/users.json\\n+++ b/public/language/en-GB/admin/manage/us ... [TRUNCATED 12136 chars] ... class=\\\"notvalidated fa fa-check text-muted\\\" title=\\\"not validated\\\">\\n+\\t\\t\\t\\t\\t\\t\\t\\t\\n \\t\\t\\t\\t\\t\\t\\t\\t[[admin/manage/users:users.no-email]]\\n \\t\\t\\t\\t\\t\\t\\t\\t{{{ end }}}\\n \\t\\t\\t\\t\\t\\t\\t\\n\",\n \"test_patch\": \"diff --git a/test/database/keys.js b/test/database/keys.js\\nindex 3941edb65a93..fde4bbc442cf 100644\\n--- a/test/database/keys.js\\n+++ b/test/database/keys.js\\n@@ -35,6 +35,17 @@ describe('Key methods', () => {\\n \\t\\t});\\n \\t});\\n \\n+\\tit('should return m ... [TRUNCATED 952 chars] ... {uid}`, 1000);\\n+\\t\\t\\tconst code = await db.get(`confirm:byUid:${uid}`);\\n+\\t\\t\\tawait db.setObjectField(`confirm:${code}`, 'expires', Date.now() + 1000);\\n \\t\\t\\tconst ok = await user.email.canSendValidation(uid, email);\\n-\\n \\t\\t\\tassert(ok);\\n \\t\\t});\\n \\t});\\n\",\n \"fail_to_pass\": \"[\\\"test/database.js | Test database test/database/keys.js::Key methods should return multiple keys and null if key doesn't exist\\\", 'test/database.js | Test database test/database/keys.js::Key methods should return empty array if keys is empty array or falsy', 'test/user/emails.js | email confirmation (library methods) canSendValidation should return true if it has been long enough to re-send confirmation']\",\n \"pass_to_pass\": \"[\\\"test/database.js | Test database should work\\\", \\\"test/database.js | Test database info should return info about database\\\", \\\"test/database.js | Test database info should not error and return info if client is falsy\\\", \\\"test/database.js | Test ... [TRUNCATED 45280 chars] ... pending or set\\\", \\\"test/user/emails.js | email confirmation (v3 api) should confirm their email (using the pending validation)\\\", \\\"test/user/emails.js | email confirmation (v3 api) should still confirm the email (as email is set in user hash)\\\"]\",\n \"before_repo_set_cmd\": \"git reset --hard 1e137b07052bc3ea0da44ed201702c94055b8ad2\\ngit clean -fd \\ngit checkout 1e137b07052bc3ea0da44ed201702c94055b8ad2 \\ngit checkout 04998908ba6721d64eba79ae3b65a351dcfbc5b5 -- test/database/keys.js test/user/emails.js\",\n \"selected_test_files_to_run\": \"[\\\"test/database.js\\\", \\\"test/database/keys.js\\\", \\\"test/user/emails.js\\\"]\",\n \"repo_language\": \"js\",\n \"requirements\": \"\\\"- The loadUserInfo(callerUid, uids) function should include logic to retrieve and attach `email:pending` and `email:expired` flags to each user object. These flags must be derived by resolving `confirm:byUid:` keys via the new `getConfi ... [TRUNCATED 3170 chars] ... rval check must compare the stored TTL timestamp if available (or, if TTL is unavailable, use the current time as the baseline) plus the configured interval against the max confirmation period, ensuring the system prevents excessive resends.\\\"\",\n \"interface\": \"\\\"Type: Method\\\\n\\\\nName: db.mget\\\\n\\\\nPath: src/database/mongo/main.js, src/database/postgres/main.js, src/database/redis/main.js\\\\n\\\\nInput: keys: string[] (An array of database keys to retrieve.)\\\\n\\\\nOutput: Promise<(string | null)[]> (A promise t ... [TRUNCATED 487 chars] ... to the email address string, or `null` if no suitable email is found.)\\\\n\\\\nDescription: A utility function that retrieves the most appropriate email address for an administrative action like \\\\\\\"force validate\\\\\\\" or \\\\\\\"resend validation email\\\\\\\".\\\"\",\n \"issue_specificity\": \"[\\\"major_bug\\\",\\\"data_bug\\\",\\\"ui_ux_bug\\\"]\",\n \"issue_categories\": \"[\\\"back_end_knowledge\\\",\\\"database_knowledge\\\",\\\"authentication_authorization_knowledge\\\",\\\"ui_ux_knowledge\\\"]\",\n \"dockerhub_tag\": \"nodebb.nodebb-NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5\",\n \"docker_image\": \"jefzda/sweap-images:nodebb.nodebb-NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `swe_bench_pro_repo_path` | `str` | `` | `scaleapi/SWE-bench_Pro-os` 仓库的本地路径。若为空，则自动克隆至 `~/.cache/evalscope/swe_bench_pro/SWE-bench_Pro-os` 并固定到 commit `ca10a60`。 |\n| `dockerhub_username` | `str` | `jefzda` | 托管 sweap-images 镜像仓库的 DockerHub 用户或组织名。 |\n| `eval_timeout` | `int` | `3600` | 每个实例的评估超时时间（秒）。 |\n\n## 使用方法\n\n### 通过 CLI 使用\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_pro \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"swe_bench_toolcall\",\"max_steps\":250}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 通过 Python 使用\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_pro'],\n agent_config=NativeAgentConfig(\n strategy='swe_bench_toolcall',\n max_steps=250,\n ),\n dataset_args={\n 'swe_bench_pro': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "098658492337015220674ea50740b5ab", "needs_translation": false }, - "updated_at": "2026-05-19T19:45:51.156530", - "translation_updated_at": "2026-05-19T19:45:55" -} \ No newline at end of file + "updated_at": "2026-07-10T20:24:34.916779", + "translation_updated_at": "2026-07-10T20:24:42" +} diff --git a/evalscope/evalscope/benchmarks/_meta/swe_bench_verified.json b/evalscope/evalscope/benchmarks/_meta/swe_bench_verified.json index 44cb7e5..d8cd23b 100644 --- a/evalscope/evalscope/benchmarks/_meta/swe_bench_verified.json +++ b/evalscope/evalscope/benchmarks/_meta/swe_bench_verified.json @@ -76,4 +76,4 @@ }, "updated_at": "2026-06-16T10:56:20.660047", "translation_updated_at": "2026-06-16T10:56:45" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/swe_bench_verified_agentic.json b/evalscope/evalscope/benchmarks/_meta/swe_bench_verified_agentic.json index 30521ed..cedfea2 100644 --- a/evalscope/evalscope/benchmarks/_meta/swe_bench_verified_agentic.json +++ b/evalscope/evalscope/benchmarks/_meta/swe_bench_verified_agentic.json @@ -15,31 +15,12 @@ "subset_list": [ "default" ], - "description": "\n## Overview\n\nSWE-bench Verified Agentic is the agentic-mode evaluation of SWE-bench Verified, a human-validated subset of 500 samples from SWE-bench. Unlike the oracle single-turn variant, the model must autonomously explore the repository, run shell commands, edit source files, and submit a patch through a multi-turn agent loop driven inside a per-instance Docker container.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic)\n- **Input**: GitHub issue description (no oracle file context)\n- **Output**: Code patch (diff format) collected from `git diff` after autonomous editing\n- **Repositories**: 12 popular Python projects (Django, Flask, Requests, etc.)\n\n## Key Features\n\n- 500 human-validated Issue-Pull Request pairs\n- Multi-turn agent loop (mini-swe-agent `swebench.yaml` compatible)\n- Per-instance SWE-bench Docker container as the execution sandbox\n- Sentinel-based patch submission protocol (`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`)\n- Supports both function-calling (`toolcall`) and text-based (`backticks`) action protocols\n\n## Evaluation Notes\n\n- Requires `pip install swebench==4.1.0` before evaluation\n- Docker images are built/pulled automatically for each repository\n- Timeout of 1800 seconds (30 min) per instance for final patch validation\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench.html) for detailed setup instructions\n- Supports both local image building and remote image pulling\n\n## Agentic Mode\n\nThis benchmark drives a multi-turn agent loop (mirrors mini-swe-agent's\n`swebench.yaml`) inside a per-instance SWE-bench Docker container. The\nmodel issues `bash` commands to explore `/testbed`, edit source files,\nand finally submits its `git diff` patch by printing the sentinel\n`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` followed by the patch contents.\n\n`extra_params.action_protocol` selects between:\n- `toolcall` (default): OpenAI function-calling protocol with a single\n `bash` tool. Recommended for any model that supports tool calling.\n- `backticks`: text-based fallback expecting one\n ` ```mswea_bash_command ``` ` block per turn. For models without\n function-calling support.\n", + "description": "\n## Overview\n\nSWE-bench Verified Agentic is the agentic-mode evaluation of SWE-bench Verified, a human-validated subset of 500 samples from SWE-bench. Unlike the oracle single-turn variant, the model must autonomously explore the repository, run shell commands, edit source files, and submit a patch through a multi-turn agent loop driven inside a per-instance Docker container.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic)\n- **Input**: GitHub issue description (no oracle file context)\n- **Output**: Code patch (diff format) collected from `git diff` after autonomous editing\n- **Repositories**: 12 popular Python projects (Django, Flask, Requests, etc.)\n\n## Key Features\n\n- 500 human-validated Issue-Pull Request pairs\n- Multi-turn agent loop (mini-swe-agent `swebench.yaml` compatible)\n- Per-instance SWE-bench Docker container as the execution sandbox\n- Sentinel-based patch submission protocol (`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`)\n- Supports both function-calling (`toolcall`) and text-based (`backticks`) action protocols\n\n## Evaluation Notes\n\n- Requires `pip install swebench==4.1.0` before evaluation\n- Docker images are built/pulled automatically for each repository\n- Timeout of 1800 seconds (30 min) per instance for final patch validation\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench.html) for detailed setup instructions\n- Supports both local image building and remote image pulling\n\n## Agentic Mode\n\nThis benchmark drives a multi-turn agent loop (mirrors mini-swe-agent's\n`swebench.yaml`) inside a per-instance SWE-bench Docker container. The\nmodel issues `bash` commands to explore `/testbed`, edit source files,\nand finally submits its `git diff` patch by printing the sentinel\n`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` followed by the patch contents.\n\nThe default `swe_bench_toolcall` strategy uses OpenAI function calling with\na single `bash` tool. Models without function-calling support can select\n`swe_bench_backticks` through `NativeAgentConfig.strategy`; that strategy\nexpects one ` ```mswea_bash_command ``` ` block per turn.\n", "prompt_template": "{question}", "system_prompt": "", "few_shot_prompt_template": "", "aggregation": "mean", "extra_params": { - "action_protocol": { - "type": "str", - "description": "Agent action protocol: \"toolcall\" (mainline OpenAI function-calling, mirrors mini-swe-agent swebench.yaml) or \"backticks\" (textbased mswea_bash_command fallback for models without function-calling support).", - "value": "toolcall", - "choices": [ - "toolcall", - "backticks" - ] - }, - "max_steps": { - "type": "int", - "description": "Maximum number of agent steps per sample.", - "value": 250 - }, - "command_timeout": { - "type": "float", - "description": "Default per-bash-command timeout in seconds.", - "value": 60.0 - }, "build_docker_images": { "type": "bool", "description": "Build Docker images locally for each sample.", @@ -67,6 +48,10 @@ } }, "sandbox_config": {}, + "agent_config": { + "strategy": "swe_bench_toolcall", + "max_steps": 250 + }, "category": "agent" }, "statistics": { @@ -158,11 +143,11 @@ "truncated": false }, "readme": { - "en": "# SWE-bench_Verified_Agentic\n\n\n## Overview\n\nSWE-bench Verified Agentic is the agentic-mode evaluation of SWE-bench Verified, a human-validated subset of 500 samples from SWE-bench. Unlike the oracle single-turn variant, the model must autonomously explore the repository, run shell commands, edit source files, and submit a patch through a multi-turn agent loop driven inside a per-instance Docker container.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic)\n- **Input**: GitHub issue description (no oracle file context)\n- **Output**: Code patch (diff format) collected from `git diff` after autonomous editing\n- **Repositories**: 12 popular Python projects (Django, Flask, Requests, etc.)\n\n## Key Features\n\n- 500 human-validated Issue-Pull Request pairs\n- Multi-turn agent loop (mini-swe-agent `swebench.yaml` compatible)\n- Per-instance SWE-bench Docker container as the execution sandbox\n- Sentinel-based patch submission protocol (`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`)\n- Supports both function-calling (`toolcall`) and text-based (`backticks`) action protocols\n\n## Evaluation Notes\n\n- Requires `pip install swebench==4.1.0` before evaluation\n- Docker images are built/pulled automatically for each repository\n- Timeout of 1800 seconds (30 min) per instance for final patch validation\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench.html) for detailed setup instructions\n- Supports both local image building and remote image pulling\n\n## Agentic Mode\n\nThis benchmark drives a multi-turn agent loop (mirrors mini-swe-agent's\n`swebench.yaml`) inside a per-instance SWE-bench Docker container. The\nmodel issues `bash` commands to explore `/testbed`, edit source files,\nand finally submits its `git diff` patch by printing the sentinel\n`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` followed by the patch contents.\n\n`extra_params.action_protocol` selects between:\n- `toolcall` (default): OpenAI function-calling protocol with a single\n `bash` tool. Recommended for any model that supports tool calling.\n- `backticks`: text-based fallback expecting one\n ` ```mswea_bash_command ``` ` block per turn. For models without\n function-calling support.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `swe_bench_verified_agentic` |\n| **Dataset ID** | [princeton-nlp/SWE-bench_Verified](https://modelscope.cn/datasets/princeton-nlp/SWE-bench_Verified/summary) |\n| **Paper** | N/A |\n| **Tags** | `Coding` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 500 |\n| Prompt Length (Mean) | 1699.73 chars |\n| Prompt Length (Min/Max) | 143 / 24770 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"360d65af\",\n \"content\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matri ... [TRUNCATED 762 chars] ... [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"problem_statement\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matri ... [TRUNCATED 762 chars] ... [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\",\n \"instance_id\": \"astropy__astropy-12907\",\n \"base_commit\": \"d16bfe05a744909de4b27f5875fe0d4ed41ce607\",\n \"patch\": \"diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\\n--- a/astropy/modeling/separable.py\\n+++ b/astropy/modeling/separable.py\\n@@ -242,7 +242,7 @@ def _cstack(left, right):\\n cright = _coord_matrix(right, 'right', noutp)\\n else:\\n cright = np.zeros((noutp, right.shape[1]))\\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\\n \\n return np.hstack([cleft, cright])\\n \\n\",\n \"PASS_TO_PASS\": [\n \"astropy/modeling/tests/test_separable.py::test_coord_matrix\",\n \"astropy/modeling/tests/test_separable.py::test_cdot\",\n \"astropy/modeling/tests/test_separable.py::test_cstack\",\n \"astropy/modeling/tests/test_separable.py::test_arith_oper\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model0-result0]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model1-result1]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model2-result2]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model3-result3]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model4-result4]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model5-result5]\",\n \"... [TRUNCATED 3 more items] ...\"\n ],\n \"FAIL_TO_PASS\": [\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model6-result6]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model9-result9]\"\n ],\n \"test_patch\": \"diff --git a/astropy/modeling/tests/test_separable.py b/astropy/modeling/tests/test_separable.py\\n--- a/astropy/modeling/tests/test_separable.py\\n+++ b/astropy/modeling/tests/test_separable.py\\n@@ -28,6 +28,13 @@\\n p1 = models.Polynomial1D(1, nam ... [TRUNCATED 931 chars] ... [True, True, False, False, False],\\n+ [False, False, True, False, False],\\n+ [False, False, False, True, False],\\n+ [False, False, False, False, True]]))),\\n }\\n \\n \\n\",\n \"version\": \"4.3\",\n \"repo\": \"astropy/astropy\",\n \"environment_setup_commit\": \"298ccb478e6bf092953bca67a3d29dc6c35f6752\",\n \"hints_text\": \"\",\n \"created_at\": \"2022-03-03T15:14:54Z\",\n \"docker_image\": \"swebench/sweb.eval.arm64.astropy_1776_astropy-12907:latest\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `action_protocol` | `str` | `toolcall` | Agent action protocol: \"toolcall\" (mainline OpenAI function-calling, mirrors mini-swe-agent swebench.yaml) or \"backticks\" (textbased mswea_bash_command fallback for models without function-calling support). Choices: ['toolcall', 'backticks'] |\n| `max_steps` | `int` | `250` | Maximum number of agent steps per sample. |\n| `command_timeout` | `float` | `60.0` | Default per-bash-command timeout in seconds. |\n| `build_docker_images` | `bool` | `True` | Build Docker images locally for each sample. |\n| `pull_remote_images_if_available` | `bool` | `True` | Attempt to pull existing remote Docker images before building. |\n| `force_arch` | `str` | `` | Optionally force a specific architecture for image build/pull. Choices: ['', 'arm64', 'x86_64'] |\n| `dockerhub_username` | `str` | `swebench` | DockerHub user/org namespace for remote SWE-bench images. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_verified_agentic \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_verified_agentic'],\n dataset_args={\n 'swe_bench_verified_agentic': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# SWE-bench_Verified_Agentic\n\n\n## 概述\n\nSWE-bench Verified Agentic 是对 SWE-bench Verified 的代理模式（agentic-mode）评估。SWE-bench Verified 是从 SWE-bench 中人工验证筛选出的 500 个样本子集。与单轮“神谕”（oracle）变体不同，模型必须在每个实例专属的 Docker 容器内，通过多轮代理循环自主探索代码仓库、执行 shell 命令、编辑源文件，并最终提交补丁。\n\n## 任务描述\n\n- **任务类型**：自动化软件工程 / 缺陷修复（代理模式）\n- **输入**：GitHub issue 描述（不提供神谕文件上下文）\n- **输出**：模型自主编辑后通过 `git diff` 生成的代码补丁（diff 格式）\n- **涉及仓库**：12 个流行的 Python 项目（如 Django、Flask、Requests 等）\n\n## 主要特性\n\n- 包含 500 个人工验证的 Issue-Pull Request 对\n- 支持多轮代理循环（兼容 mini-swe-agent 的 `swebench.yaml` 配置）\n- 每个实例使用独立的 SWE-bench Docker 容器作为执行沙箱\n- 基于哨兵值（sentinel）的补丁提交协议（`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`）\n- 同时支持函数调用（`toolcall`）和基于文本（`backticks`）的动作协议\n\n## 评估说明\n\n- 评估前需安装 `pip install swebench==4.1.0`\n- 每个仓库的 Docker 镜像会自动构建或拉取\n- 每个实例的最终补丁验证超时时间为 1800 秒（30 分钟）\n- 详细设置说明请参阅 [使用文档](https://evalscope.readthedocs.io/zh-cn/latest/third_party/swe_bench.html)\n- 支持本地构建镜像和远程拉取镜像两种方式\n\n## 代理模式\n\n该基准测试在每个实例专属的 SWE-bench Docker 容器内驱动一个多轮代理循环（与 mini-swe-agent 的 `swebench.yaml` 配置一致）。模型通过发出 `bash` 命令来探索 `/testbed` 目录、编辑源文件，并最终通过打印哨兵值 `COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` 及其后的补丁内容来提交 `git diff` 补丁。\n\n`extra_params.action_protocol` 参数用于选择以下两种协议之一：\n- `toolcall`（默认）：采用 OpenAI 函数调用协议，仅提供一个 `bash` 工具。推荐用于支持工具调用的模型。\n- `backticks`：基于文本的备用协议，要求每轮输出一个 ` ```mswea_bash_command ``` ` 代码块。适用于不支持函数调用的模型。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `swe_bench_verified_agentic` |\n| **数据集ID** | [princeton-nlp/SWE-bench_Verified](https://modelscope.cn/datasets/princeton-nlp/SWE-bench_Verified/summary) |\n| **论文** | N/A |\n| **标签** | `Coding` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 500 |\n| 提示词长度（平均） | 1699.73 字符 |\n| 提示词长度（最小/最大） | 143 / 24770 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"360d65af\",\n \"content\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matri ... [TRUNCATED 762 chars] ... [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"problem_statement\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matri ... [TRUNCATED 762 chars] ... [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\",\n \"instance_id\": \"astropy__astropy-12907\",\n \"base_commit\": \"d16bfe05a744909de4b27f5875fe0d4ed41ce607\",\n \"patch\": \"diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\\n--- a/astropy/modeling/separable.py\\n+++ b/astropy/modeling/separable.py\\n@@ -242,7 +242,7 @@ def _cstack(left, right):\\n cright = _coord_matrix(right, 'right', noutp)\\n else:\\n cright = np.zeros((noutp, right.shape[1]))\\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\\n \\n return np.hstack([cleft, cright])\\n \\n\",\n \"PASS_TO_PASS\": [\n \"astropy/modeling/tests/test_separable.py::test_coord_matrix\",\n \"astropy/modeling/tests/test_separable.py::test_cdot\",\n \"astropy/modeling/tests/test_separable.py::test_cstack\",\n \"astropy/modeling/tests/test_separable.py::test_arith_oper\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model0-result0]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model1-result1]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model2-result2]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model3-result3]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model4-result4]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model5-result5]\",\n \"... [TRUNCATED 3 more items] ...\"\n ],\n \"FAIL_TO_PASS\": [\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model6-result6]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model9-result9]\"\n ],\n \"test_patch\": \"diff --git a/astropy/modeling/tests/test_separable.py b/astropy/modeling/tests/test_separable.py\\n--- a/astropy/modeling/tests/test_separable.py\\n+++ b/astropy/modeling/tests/test_separable.py\\n@@ -28,6 +28,13 @@\\n p1 = models.Polynomial1D(1, nam ... [TRUNCATED 931 chars] ... [True, True, False, False, False],\\n+ [False, False, True, False, False],\\n+ [False, False, False, True, False],\\n+ [False, False, False, False, True]]))),\\n }\\n \\n \\n\",\n \"version\": \"4.3\",\n \"repo\": \"astropy/astropy\",\n \"environment_setup_commit\": \"298ccb478e6bf092953bca67a3d29dc6c35f6752\",\n \"hints_text\": \"\",\n \"created_at\": \"2022-03-03T15:14:54Z\",\n \"docker_image\": \"swebench/sweb.eval.arm64.astropy_1776_astropy-12907:latest\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `action_protocol` | `str` | `toolcall` | 代理动作协议：\"toolcall\"（主流 OpenAI 函数调用方式，与 mini-swe-agent 的 swebench.yaml 一致）或 \"backticks\"（针对不支持函数调用的模型的基于文本的 mswea_bash_command 回退方案）。可选值：['toolcall', 'backticks'] |\n| `max_steps` | `int` | `250` | 每个样本的最大代理步数。 |\n| `command_timeout` | `float` | `60.0` | 每个 bash 命令的默认超时时间（秒）。 |\n| `build_docker_images` | `bool` | `True` | 是否为每个样本在本地构建 Docker 镜像。 |\n| `pull_remote_images_if_available` | `bool` | `True` | 在构建前是否尝试拉取已存在的远程 Docker 镜像。 |\n| `force_arch` | `str` | `` | 可选地强制指定镜像构建/拉取的目标架构。可选值：['', 'arm64', 'x86_64'] |\n| `dockerhub_username` | `str` | `swebench` | 远程 SWE-bench 镜像在 DockerHub 上的用户/组织命名空间。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_verified_agentic \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_verified_agentic'],\n dataset_args={\n 'swe_bench_verified_agentic': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "f6ea67447e7a0421b71c67f6987a7925", + "en": "# SWE-bench_Verified_Agentic\n\n\n## Overview\n\nSWE-bench Verified Agentic is the agentic-mode evaluation of SWE-bench Verified, a human-validated subset of 500 samples from SWE-bench. Unlike the oracle single-turn variant, the model must autonomously explore the repository, run shell commands, edit source files, and submit a patch through a multi-turn agent loop driven inside a per-instance Docker container.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic)\n- **Input**: GitHub issue description (no oracle file context)\n- **Output**: Code patch (diff format) collected from `git diff` after autonomous editing\n- **Repositories**: 12 popular Python projects (Django, Flask, Requests, etc.)\n\n## Key Features\n\n- 500 human-validated Issue-Pull Request pairs\n- Multi-turn agent loop (mini-swe-agent `swebench.yaml` compatible)\n- Per-instance SWE-bench Docker container as the execution sandbox\n- Sentinel-based patch submission protocol (`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`)\n- Supports both function-calling (`toolcall`) and text-based (`backticks`) action protocols\n\n## Evaluation Notes\n\n- Requires `pip install swebench==4.1.0` before evaluation\n- Docker images are built/pulled automatically for each repository\n- Timeout of 1800 seconds (30 min) per instance for final patch validation\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench.html) for detailed setup instructions\n- Supports both local image building and remote image pulling\n\n## Agentic Mode\n\nThis benchmark drives a multi-turn agent loop (mirrors mini-swe-agent's\n`swebench.yaml`) inside a per-instance SWE-bench Docker container. The\nmodel issues `bash` commands to explore `/testbed`, edit source files,\nand finally submits its `git diff` patch by printing the sentinel\n`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` followed by the patch contents.\n\nThe default `swe_bench_toolcall` strategy uses OpenAI function calling with\na single `bash` tool. Models without function-calling support can select\n`swe_bench_backticks` through `NativeAgentConfig.strategy`; that strategy\nexpects one ` ```mswea_bash_command ``` ` block per turn.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `swe_bench_verified_agentic` |\n| **Dataset ID** | [princeton-nlp/SWE-bench_Verified](https://modelscope.cn/datasets/princeton-nlp/SWE-bench_Verified/summary) |\n| **Paper** | N/A |\n| **Tags** | `Coding` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 500 |\n| Prompt Length (Mean) | 1699.73 chars |\n| Prompt Length (Min/Max) | 143 / 24770 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"360d65af\",\n \"content\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matri ... [TRUNCATED 762 chars] ... [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"problem_statement\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matri ... [TRUNCATED 762 chars] ... [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\",\n \"instance_id\": \"astropy__astropy-12907\",\n \"base_commit\": \"d16bfe05a744909de4b27f5875fe0d4ed41ce607\",\n \"patch\": \"diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\\n--- a/astropy/modeling/separable.py\\n+++ b/astropy/modeling/separable.py\\n@@ -242,7 +242,7 @@ def _cstack(left, right):\\n cright = _coord_matrix(right, 'right', noutp)\\n else:\\n cright = np.zeros((noutp, right.shape[1]))\\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\\n \\n return np.hstack([cleft, cright])\\n \\n\",\n \"PASS_TO_PASS\": [\n \"astropy/modeling/tests/test_separable.py::test_coord_matrix\",\n \"astropy/modeling/tests/test_separable.py::test_cdot\",\n \"astropy/modeling/tests/test_separable.py::test_cstack\",\n \"astropy/modeling/tests/test_separable.py::test_arith_oper\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model0-result0]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model1-result1]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model2-result2]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model3-result3]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model4-result4]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model5-result5]\",\n \"... [TRUNCATED 3 more items] ...\"\n ],\n \"FAIL_TO_PASS\": [\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model6-result6]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model9-result9]\"\n ],\n \"test_patch\": \"diff --git a/astropy/modeling/tests/test_separable.py b/astropy/modeling/tests/test_separable.py\\n--- a/astropy/modeling/tests/test_separable.py\\n+++ b/astropy/modeling/tests/test_separable.py\\n@@ -28,6 +28,13 @@\\n p1 = models.Polynomial1D(1, nam ... [TRUNCATED 931 chars] ... [True, True, False, False, False],\\n+ [False, False, True, False, False],\\n+ [False, False, False, True, False],\\n+ [False, False, False, False, True]]))),\\n }\\n \\n \\n\",\n \"version\": \"4.3\",\n \"repo\": \"astropy/astropy\",\n \"environment_setup_commit\": \"298ccb478e6bf092953bca67a3d29dc6c35f6752\",\n \"hints_text\": \"\",\n \"created_at\": \"2022-03-03T15:14:54Z\",\n \"docker_image\": \"swebench/sweb.eval.arm64.astropy_1776_astropy-12907:latest\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `build_docker_images` | `bool` | `True` | Build Docker images locally for each sample. |\n| `pull_remote_images_if_available` | `bool` | `True` | Attempt to pull existing remote Docker images before building. |\n| `force_arch` | `str` | `` | Optionally force a specific architecture for image build/pull. Choices: ['', 'arm64', 'x86_64'] |\n| `dockerhub_username` | `str` | `swebench` | DockerHub user/org namespace for remote SWE-bench images. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_verified_agentic \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"swe_bench_toolcall\",\"max_steps\":250}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_verified_agentic'],\n agent_config=NativeAgentConfig(\n strategy='swe_bench_toolcall',\n max_steps=250,\n ),\n dataset_args={\n 'swe_bench_verified_agentic': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# SWE-bench_Verified_Agentic\n\n\n## 概述\n\nSWE-bench Verified Agentic 是对 SWE-bench Verified 的代理模式（agentic-mode）评估，后者是从 SWE-bench 中人工验证筛选出的 500 个样本子集。与提供 oracle 文件上下文的单轮变体不同，模型必须在每个实例专属的 Docker 容器内，通过多轮代理循环自主探索代码仓库、执行 shell 命令、编辑源文件，并最终提交补丁。\n\n## 任务描述\n\n- **任务类型**：自动化软件工程 / 缺陷修复（代理模式）\n- **输入**：GitHub issue 描述（无 oracle 文件上下文）\n- **输出**：自主编辑后通过 `git diff` 收集的代码补丁（diff 格式）\n- **涉及仓库**：12 个流行的 Python 项目（如 Django、Flask、Requests 等）\n\n## 主要特性\n\n- 包含 500 个人工验证的 Issue-Pull Request 对\n- 支持多轮代理循环（兼容 mini-swe-agent 的 `swebench.yaml` 配置）\n- 每个实例使用独立的 SWE-bench Docker 容器作为执行沙箱\n- 基于哨兵（sentinel）的补丁提交协议（`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`）\n- 同时支持函数调用（`toolcall`）和基于文本（`backticks`）的动作协议\n\n## 评估说明\n\n- 评估前需安装 `pip install swebench==4.1.0`\n- 每个仓库的 Docker 镜像会自动构建或拉取\n- 每个实例的最终补丁验证超时时间为 1800 秒（30 分钟）\n- 详细设置说明请参阅 [使用文档](https://evalscope.readthedocs.io/zh-cn/latest/third_party/swe_bench.html)\n- 支持本地构建镜像和远程拉取镜像两种方式\n\n## 代理模式\n\n该基准测试在每个实例专属的 SWE-bench Docker 容器内驱动一个多轮代理循环（与 mini-swe-agent 的 `swebench.yaml` 配置一致）。模型通过发出 `bash` 命令来探索 `/testbed` 目录、编辑源文件，并最终通过打印哨兵字符串 `COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` 及其后的补丁内容来提交 `git diff` 补丁。\n\n默认的 `swe_bench_toolcall` 策略使用 OpenAI 函数调用，仅提供一个 `bash` 工具。不支持函数调用的模型可通过 `NativeAgentConfig.strategy` 选择 `swe_bench_backticks` 策略；该策略要求每轮输出一个 ` ```mswea_bash_command ``` ` 代码块。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `swe_bench_verified_agentic` |\n| **数据集ID** | [princeton-nlp/SWE-bench_Verified](https://modelscope.cn/datasets/princeton-nlp/SWE-bench_Verified/summary) |\n| **论文** | N/A |\n| **标签** | `Coding` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 500 |\n| 提示词长度（平均） | 1699.73 字符 |\n| 提示词长度（最小/最大） | 143 / 24770 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"360d65af\",\n \"content\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matri ... [TRUNCATED 762 chars] ... [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"problem_statement\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matri ... [TRUNCATED 762 chars] ... [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\",\n \"instance_id\": \"astropy__astropy-12907\",\n \"base_commit\": \"d16bfe05a744909de4b27f5875fe0d4ed41ce607\",\n \"patch\": \"diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\\n--- a/astropy/modeling/separable.py\\n+++ b/astropy/modeling/separable.py\\n@@ -242,7 +242,7 @@ def _cstack(left, right):\\n cright = _coord_matrix(right, 'right', noutp)\\n else:\\n cright = np.zeros((noutp, right.shape[1]))\\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\\n \\n return np.hstack([cleft, cright])\\n \\n\",\n \"PASS_TO_PASS\": [\n \"astropy/modeling/tests/test_separable.py::test_coord_matrix\",\n \"astropy/modeling/tests/test_separable.py::test_cdot\",\n \"astropy/modeling/tests/test_separable.py::test_cstack\",\n \"astropy/modeling/tests/test_separable.py::test_arith_oper\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model0-result0]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model1-result1]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model2-result2]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model3-result3]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model4-result4]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model5-result5]\",\n \"... [TRUNCATED 3 more items] ...\"\n ],\n \"FAIL_TO_PASS\": [\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model6-result6]\",\n \"astropy/modeling/tests/test_separable.py::test_separable[compound_model9-result9]\"\n ],\n \"test_patch\": \"diff --git a/astropy/modeling/tests/test_separable.py b/astropy/modeling/tests/test_separable.py\\n--- a/astropy/modeling/tests/test_separable.py\\n+++ b/astropy/modeling/tests/test_separable.py\\n@@ -28,6 +28,13 @@\\n p1 = models.Polynomial1D(1, nam ... [TRUNCATED 931 chars] ... [True, True, False, False, False],\\n+ [False, False, True, False, False],\\n+ [False, False, False, True, False],\\n+ [False, False, False, False, True]]))),\\n }\\n \\n \\n\",\n \"version\": \"4.3\",\n \"repo\": \"astropy/astropy\",\n \"environment_setup_commit\": \"298ccb478e6bf092953bca67a3d29dc6c35f6752\",\n \"hints_text\": \"\",\n \"created_at\": \"2022-03-03T15:14:54Z\",\n \"docker_image\": \"swebench/sweb.eval.arm64.astropy_1776_astropy-12907:latest\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板:**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `build_docker_images` | `bool` | `True` | 为每个样本在本地构建 Docker 镜像。 |\n| `pull_remote_images_if_available` | `bool` | `True` | 在构建前尝试拉取已存在的远程 Docker 镜像。 |\n| `force_arch` | `str` | `` | 可选地强制指定镜像构建/拉取的架构。可选项：['', 'arm64', 'x86_64'] |\n| `dockerhub_username` | `str` | `swebench` | 远程 SWE-bench 镜像在 DockerHub 上的用户/组织命名空间。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_verified_agentic \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"swe_bench_toolcall\",\"max_steps\":250}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_verified_agentic'],\n agent_config=NativeAgentConfig(\n strategy='swe_bench_toolcall',\n max_steps=250,\n ),\n dataset_args={\n 'swe_bench_verified_agentic': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "16b9fbe9ce19a047d01f0e8fc06b2c7a", "needs_translation": false }, - "updated_at": "2026-06-16T10:56:20.660371", - "translation_updated_at": "2026-06-16T10:56:45" -} \ No newline at end of file + "updated_at": "2026-07-10T20:24:34.529526", + "translation_updated_at": "2026-07-10T20:24:42" +} diff --git a/evalscope/evalscope/benchmarks/_meta/swe_bench_verified_mini.json b/evalscope/evalscope/benchmarks/_meta/swe_bench_verified_mini.json index 50b9b1c..79c5b63 100644 --- a/evalscope/evalscope/benchmarks/_meta/swe_bench_verified_mini.json +++ b/evalscope/evalscope/benchmarks/_meta/swe_bench_verified_mini.json @@ -76,4 +76,4 @@ }, "updated_at": "2026-06-16T10:56:20.659995", "translation_updated_at": "2026-06-16T10:56:45" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/swe_bench_verified_mini_agentic.json b/evalscope/evalscope/benchmarks/_meta/swe_bench_verified_mini_agentic.json index 3629335..54230ca 100644 --- a/evalscope/evalscope/benchmarks/_meta/swe_bench_verified_mini_agentic.json +++ b/evalscope/evalscope/benchmarks/_meta/swe_bench_verified_mini_agentic.json @@ -15,31 +15,12 @@ "subset_list": [ "default" ], - "description": "\n## Overview\n\nSWE-bench Verified Mini Agentic is the agentic-mode evaluation of SWE-bench Verified Mini, a compact 50-sample subset that maintains the same distribution of performance, test pass rates, and difficulty as the full Verified set while requiring only 5GB of storage instead of 130GB. The model must autonomously explore, edit, and submit a patch through a multi-turn agent loop.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic)\n- **Input**: GitHub issue description (no oracle file context)\n- **Output**: Code patch (diff format) collected from `git diff` after autonomous editing\n- **Size**: 50 samples (vs 500 in full Verified set)\n\n## Key Features\n\n- Representative 50-sample subset of SWE-bench Verified\n- Same difficulty distribution as the full dataset\n- Dramatically reduced storage requirements (5GB vs 130GB)\n- Multi-turn agent loop with per-instance Docker sandbox\n- Ideal for quick agentic evaluation and development iteration\n\n## Evaluation Notes\n\n- Requires `pip install swebench==4.1.0` before evaluation\n- Docker images are built/pulled automatically\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench.html) for detailed setup\n- Good for rapid prototyping of agent strategies and initial model assessment\n\n## Agentic Mode\n\nThis benchmark drives a multi-turn agent loop (mirrors mini-swe-agent's\n`swebench.yaml`) inside a per-instance SWE-bench Docker container. The\nmodel issues `bash` commands to explore `/testbed`, edit source files,\nand finally submits its `git diff` patch by printing the sentinel\n`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` followed by the patch contents.\n\n`extra_params.action_protocol` selects between:\n- `toolcall` (default): OpenAI function-calling protocol with a single\n `bash` tool. Recommended for any model that supports tool calling.\n- `backticks`: text-based fallback expecting one\n ` ```mswea_bash_command ``` ` block per turn. For models without\n function-calling support.\n", + "description": "\n## Overview\n\nSWE-bench Verified Mini Agentic is the agentic-mode evaluation of SWE-bench Verified Mini, a compact 50-sample subset that maintains the same distribution of performance, test pass rates, and difficulty as the full Verified set while requiring only 5GB of storage instead of 130GB. The model must autonomously explore, edit, and submit a patch through a multi-turn agent loop.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic)\n- **Input**: GitHub issue description (no oracle file context)\n- **Output**: Code patch (diff format) collected from `git diff` after autonomous editing\n- **Size**: 50 samples (vs 500 in full Verified set)\n\n## Key Features\n\n- Representative 50-sample subset of SWE-bench Verified\n- Same difficulty distribution as the full dataset\n- Dramatically reduced storage requirements (5GB vs 130GB)\n- Multi-turn agent loop with per-instance Docker sandbox\n- Ideal for quick agentic evaluation and development iteration\n\n## Evaluation Notes\n\n- Requires `pip install swebench==4.1.0` before evaluation\n- Docker images are built/pulled automatically\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench.html) for detailed setup\n- Good for rapid prototyping of agent strategies and initial model assessment\n\n## Agentic Mode\n\nThis benchmark drives a multi-turn agent loop (mirrors mini-swe-agent's\n`swebench.yaml`) inside a per-instance SWE-bench Docker container. The\nmodel issues `bash` commands to explore `/testbed`, edit source files,\nand finally submits its `git diff` patch by printing the sentinel\n`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` followed by the patch contents.\n\nThe default `swe_bench_toolcall` strategy uses OpenAI function calling with\na single `bash` tool. Models without function-calling support can select\n`swe_bench_backticks` through `NativeAgentConfig.strategy`; that strategy\nexpects one ` ```mswea_bash_command ``` ` block per turn.\n", "prompt_template": "{question}", "system_prompt": "", "few_shot_prompt_template": "", "aggregation": "mean", "extra_params": { - "action_protocol": { - "type": "str", - "description": "Agent action protocol: \"toolcall\" (mainline OpenAI function-calling, mirrors mini-swe-agent swebench.yaml) or \"backticks\" (textbased mswea_bash_command fallback for models without function-calling support).", - "value": "toolcall", - "choices": [ - "toolcall", - "backticks" - ] - }, - "max_steps": { - "type": "int", - "description": "Maximum number of agent steps per sample.", - "value": 250 - }, - "command_timeout": { - "type": "float", - "description": "Default per-bash-command timeout in seconds.", - "value": 60.0 - }, "build_docker_images": { "type": "bool", "description": "Build Docker images locally for each sample.", @@ -67,6 +48,10 @@ } }, "sandbox_config": {}, + "agent_config": { + "strategy": "swe_bench_toolcall", + "max_steps": 250 + }, "category": "agent" }, "statistics": { @@ -158,11 +143,11 @@ "truncated": false }, "readme": { - "en": "# SWE-bench_Verified_Mini_Agentic\n\n\n## Overview\n\nSWE-bench Verified Mini Agentic is the agentic-mode evaluation of SWE-bench Verified Mini, a compact 50-sample subset that maintains the same distribution of performance, test pass rates, and difficulty as the full Verified set while requiring only 5GB of storage instead of 130GB. The model must autonomously explore, edit, and submit a patch through a multi-turn agent loop.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic)\n- **Input**: GitHub issue description (no oracle file context)\n- **Output**: Code patch (diff format) collected from `git diff` after autonomous editing\n- **Size**: 50 samples (vs 500 in full Verified set)\n\n## Key Features\n\n- Representative 50-sample subset of SWE-bench Verified\n- Same difficulty distribution as the full dataset\n- Dramatically reduced storage requirements (5GB vs 130GB)\n- Multi-turn agent loop with per-instance Docker sandbox\n- Ideal for quick agentic evaluation and development iteration\n\n## Evaluation Notes\n\n- Requires `pip install swebench==4.1.0` before evaluation\n- Docker images are built/pulled automatically\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench.html) for detailed setup\n- Good for rapid prototyping of agent strategies and initial model assessment\n\n## Agentic Mode\n\nThis benchmark drives a multi-turn agent loop (mirrors mini-swe-agent's\n`swebench.yaml`) inside a per-instance SWE-bench Docker container. The\nmodel issues `bash` commands to explore `/testbed`, edit source files,\nand finally submits its `git diff` patch by printing the sentinel\n`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` followed by the patch contents.\n\n`extra_params.action_protocol` selects between:\n- `toolcall` (default): OpenAI function-calling protocol with a single\n `bash` tool. Recommended for any model that supports tool calling.\n- `backticks`: text-based fallback expecting one\n ` ```mswea_bash_command ``` ` block per turn. For models without\n function-calling support.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `swe_bench_verified_mini_agentic` |\n| **Dataset ID** | [evalscope/swe-bench-verified-mini](https://modelscope.cn/datasets/evalscope/swe-bench-verified-mini/summary) |\n| **Paper** | N/A |\n| **Tags** | `Coding` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 50 |\n| Prompt Length (Mean) | 1268.5 chars |\n| Prompt Length (Min/Max) | 257 / 5362 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"363bb967\",\n \"content\": \"AuthenticationForm's username field doesn't set maxlength HTML attribute.\\nDescription\\n\\t\\nAuthenticationForm's username field doesn't render with maxlength HTML attribute anymore.\\nRegression introduced in #27515 and 5ceaf14686ce626404afb6a5fbd3d8286410bf13.\\n​https://groups.google.com/forum/?utm_source=digest&utm_medium=email#!topic/django-developers/qnfSqro0DlA\\n​https://forum.djangoproject.com/t/possible-authenticationform-max-length-regression-in-django-2-1/241\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"problem_statement\": \"AuthenticationForm's username field doesn't set maxlength HTML attribute.\\nDescription\\n\\t\\nAuthenticationForm's username field doesn't render with maxlength HTML attribute anymore.\\nRegression introduced in #27515 and 5ceaf14686ce626404afb6a5fbd3d8286410bf13.\\n​https://groups.google.com/forum/?utm_source=digest&utm_medium=email#!topic/django-developers/qnfSqro0DlA\\n​https://forum.djangoproject.com/t/possible-authenticationform-max-length-regression-in-django-2-1/241\\n\",\n \"instance_id\": \"django__django-11790\",\n \"base_commit\": \"b1d6b35e146aea83b171c1b921178bbaae2795ed\",\n \"patch\": \"diff --git a/django/contrib/auth/forms.py b/django/contrib/auth/forms.py\\n--- a/django/contrib/auth/forms.py\\n+++ b/django/contrib/auth/forms.py\\n@@ -191,7 +191,9 @@ def __init__(self, request=None, *args, **kwargs):\\n \\n # Set the max len ... [TRUNCATED 322 chars] ... username_max_length\\n+ self.fields['username'].widget.attrs['maxlength'] = username_max_length\\n if self.fields['username'].label is None:\\n self.fields['username'].label = capfirst(self.username_field.verbose_name)\\n \\n\",\n \"PASS_TO_PASS\": [\n \"test_html_autocomplete_attributes (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_missing_passwords (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_non_matching_passwords (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_one_password (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_password_whitespace_not_stripped (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_success (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_field_order (auth_tests.test_forms.PasswordChangeFormTest)\",\n \"test_html_autocomplete_attributes (auth_tests.test_forms.PasswordChangeFormTest)\",\n \"test_incorrect_password (auth_tests.test_forms.PasswordChangeFormTest)\",\n \"test_password_verification (auth_tests.test_forms.PasswordChangeFormTest)\",\n \"... [TRUNCATED 67 more items] ...\"\n ],\n \"FAIL_TO_PASS\": [\n \"test_username_field_max_length_defaults_to_254 (auth_tests.test_forms.AuthenticationFormTest)\",\n \"test_username_field_max_length_matches_user_model (auth_tests.test_forms.AuthenticationFormTest)\"\n ],\n \"test_patch\": \"diff --git a/tests/auth_tests/test_forms.py b/tests/auth_tests/test_forms.py\\n--- a/tests/auth_tests/test_forms.py\\n+++ b/tests/auth_tests/test_forms.py\\n@@ -423,6 +423,7 @@ def test_username_field_max_length_matches_user_model(self):\\n C ... [TRUNCATED 543 chars] ... )\\n self.assertEqual(form.fields['username'].max_length, 254)\\n+ self.assertEqual(form.fields['username'].widget.attrs.get('maxlength'), 254)\\n self.assertEqual(form.errors, {})\\n \\n def test_username_field_label(self):\\n\",\n \"version\": \"3.1\",\n \"repo\": \"django/django\",\n \"environment_setup_commit\": \"0668164b4ac93a5be79f5b87fae83c657124d9ab\",\n \"hints_text\": \"Regression test.\",\n \"created_at\": \"2019-09-17T14:33:44Z\",\n \"docker_image\": \"swebench/sweb.eval.arm64.django_1776_django-11790:latest\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `action_protocol` | `str` | `toolcall` | Agent action protocol: \"toolcall\" (mainline OpenAI function-calling, mirrors mini-swe-agent swebench.yaml) or \"backticks\" (textbased mswea_bash_command fallback for models without function-calling support). Choices: ['toolcall', 'backticks'] |\n| `max_steps` | `int` | `250` | Maximum number of agent steps per sample. |\n| `command_timeout` | `float` | `60.0` | Default per-bash-command timeout in seconds. |\n| `build_docker_images` | `bool` | `True` | Build Docker images locally for each sample. |\n| `pull_remote_images_if_available` | `bool` | `True` | Attempt to pull existing remote Docker images before building. |\n| `force_arch` | `str` | `` | Optionally force a specific architecture for image build/pull. Choices: ['', 'arm64', 'x86_64'] |\n| `dockerhub_username` | `str` | `swebench` | DockerHub user/org namespace for remote SWE-bench images. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_verified_mini_agentic \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_verified_mini_agentic'],\n dataset_args={\n 'swe_bench_verified_mini_agentic': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# SWE-bench_Verified_Mini_Agentic\n\n\n## 概述\n\nSWE-bench Verified Mini Agentic 是对 SWE-bench Verified Mini 的代理模式（agentic-mode）评估。SWE-bench Verified Mini 是一个精简的 50 样本子集，在保持与完整 Verified 集合相同的性能分布、测试通过率和难度的同时，仅需 5GB 存储空间（而非 130GB）。模型必须通过多轮代理循环，自主探索、编辑代码并提交补丁。\n\n## 任务描述\n\n- **任务类型**：自动化软件工程 / 缺陷修复（代理模式）\n- **输入**：GitHub issue 描述（无 oracle 文件上下文）\n- **输出**：代码补丁（以 `git diff` 生成的 diff 格式）\n- **规模**：50 个样本（完整 Verified 集合为 500 个）\n\n## 主要特性\n\n- SWE-bench Verified 的代表性 50 样本子集\n- 与完整数据集具有相同的难度分布\n- 存储需求大幅降低（5GB vs 130GB）\n- 每个实例使用独立 Docker 沙箱的多轮代理循环\n- 非常适合快速代理评估和开发迭代\n\n## 评估说明\n\n- 评估前需执行 `pip install swebench==4.1.0`\n- Docker 镜像会自动构建或拉取\n- 详细设置请参阅 [使用文档](https://evalscope.readthedocs.io/zh-cn/latest/third_party/swe_bench.html)\n- 适用于代理策略的快速原型设计和模型初步评估\n\n## 代理模式\n\n该基准测试在每个实例专属的 SWE-bench Docker 容器内驱动一个多轮代理循环（与 mini-swe-agent 的 `swebench.yaml` 配置一致）。模型通过发出 `bash` 命令来探索 `/testbed` 目录、编辑源文件，并最终通过打印哨兵字符串 `COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` 及其后的补丁内容来提交 `git diff` 补丁。\n\n`extra_params.action_protocol` 可选择以下两种协议：\n- `toolcall`（默认）：OpenAI 函数调用协议，仅提供一个 `bash` 工具。推荐用于支持工具调用的模型。\n- `backticks`：基于文本的备用协议，每轮期望一个 ` ```mswea_bash_command ``` ` 代码块。适用于不支持函数调用的模型。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `swe_bench_verified_mini_agentic` |\n| **数据集ID** | [evalscope/swe-bench-verified-mini](https://modelscope.cn/datasets/evalscope/swe-bench-verified-mini/summary) |\n| **论文** | N/A |\n| **标签** | `Coding` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 50 |\n| 提示词长度（平均） | 1268.5 字符 |\n| 提示词长度（最小/最大） | 257 / 5362 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"363bb967\",\n \"content\": \"AuthenticationForm's username field doesn't set maxlength HTML attribute.\\nDescription\\n\\t\\nAuthenticationForm's username field doesn't render with maxlength HTML attribute anymore.\\nRegression introduced in #27515 and 5ceaf14686ce626404afb6a5fbd3d8286410bf13.\\n​https://groups.google.com/forum/?utm_source=digest&utm_medium=email#!topic/django-developers/qnfSqro0DlA\\n​https://forum.djangoproject.com/t/possible-authenticationform-max-length-regression-in-django-2-1/241\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"problem_statement\": \"AuthenticationForm's username field doesn't set maxlength HTML attribute.\\nDescription\\n\\t\\nAuthenticationForm's username field doesn't render with maxlength HTML attribute anymore.\\nRegression introduced in #27515 and 5ceaf14686ce626404afb6a5fbd3d8286410bf13.\\n​https://groups.google.com/forum/?utm_source=digest&utm_medium=email#!topic/django-developers/qnfSqro0DlA\\n​https://forum.djangoproject.com/t/possible-authenticationform-max-length-regression-in-django-2-1/241\\n\",\n \"instance_id\": \"django__django-11790\",\n \"base_commit\": \"b1d6b35e146aea83b171c1b921178bbaae2795ed\",\n \"patch\": \"diff --git a/django/contrib/auth/forms.py b/django/contrib/auth/forms.py\\n--- a/django/contrib/auth/forms.py\\n+++ b/django/contrib/auth/forms.py\\n@@ -191,7 +191,9 @@ def __init__(self, request=None, *args, **kwargs):\\n \\n # Set the max len ... [TRUNCATED 322 chars] ... username_max_length\\n+ self.fields['username'].widget.attrs['maxlength'] = username_max_length\\n if self.fields['username'].label is None:\\n self.fields['username'].label = capfirst(self.username_field.verbose_name)\\n \\n\",\n \"PASS_TO_PASS\": [\n \"test_html_autocomplete_attributes (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_missing_passwords (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_non_matching_passwords (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_one_password (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_password_whitespace_not_stripped (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_success (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_field_order (auth_tests.test_forms.PasswordChangeFormTest)\",\n \"test_html_autocomplete_attributes (auth_tests.test_forms.PasswordChangeFormTest)\",\n \"test_incorrect_password (auth_tests.test_forms.PasswordChangeFormTest)\",\n \"test_password_verification (auth_tests.test_forms.PasswordChangeFormTest)\",\n \"... [TRUNCATED 67 more items] ...\"\n ],\n \"FAIL_TO_PASS\": [\n \"test_username_field_max_length_defaults_to_254 (auth_tests.test_forms.AuthenticationFormTest)\",\n \"test_username_field_max_length_matches_user_model (auth_tests.test_forms.AuthenticationFormTest)\"\n ],\n \"test_patch\": \"diff --git a/tests/auth_tests/test_forms.py b/tests/auth_tests/test_forms.py\\n--- a/tests/auth_tests/test_forms.py\\n+++ b/tests/auth_tests/test_forms.py\\n@@ -423,6 +423,7 @@ def test_username_field_max_length_matches_user_model(self):\\n C ... [TRUNCATED 543 chars] ... )\\n self.assertEqual(form.fields['username'].max_length, 254)\\n+ self.assertEqual(form.fields['username'].widget.attrs.get('maxlength'), 254)\\n self.assertEqual(form.errors, {})\\n \\n def test_username_field_label(self):\\n\",\n \"version\": \"3.1\",\n \"repo\": \"django/django\",\n \"environment_setup_commit\": \"0668164b4ac93a5be79f5b87fae83c657124d9ab\",\n \"hints_text\": \"Regression test.\",\n \"created_at\": \"2019-09-17T14:33:44Z\",\n \"docker_image\": \"swebench/sweb.eval.arm64.django_1776_django-11790:latest\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `action_protocol` | `str` | `toolcall` | 代理动作协议：\"toolcall\"（主流 OpenAI 函数调用方式，与 mini-swe-agent 的 swebench.yaml 一致）或 \"backticks\"（针对不支持函数调用的模型的文本式 mswea_bash_command 回退方案）。可选值：['toolcall', 'backticks'] |\n| `max_steps` | `int` | `250` | 每个样本的最大代理步数。 |\n| `command_timeout` | `float` | `60.0` | 每个 bash 命令的默认超时时间（秒）。 |\n| `build_docker_images` | `bool` | `True` | 为每个样本在本地构建 Docker 镜像。 |\n| `pull_remote_images_if_available` | `bool` | `True` | 在构建前尝试拉取已存在的远程 Docker 镜像。 |\n| `force_arch` | `str` | `` | 可选地强制指定镜像构建/拉取的架构。可选值：['', 'arm64', 'x86_64'] |\n| `dockerhub_username` | `str` | `swebench` | 远程 SWE-bench 镜像的 DockerHub 用户/组织命名空间。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_verified_mini_agentic \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_verified_mini_agentic'],\n dataset_args={\n 'swe_bench_verified_mini_agentic': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "0f8056724238f62bc7de991132e04b38", + "en": "# SWE-bench_Verified_Mini_Agentic\n\n\n## Overview\n\nSWE-bench Verified Mini Agentic is the agentic-mode evaluation of SWE-bench Verified Mini, a compact 50-sample subset that maintains the same distribution of performance, test pass rates, and difficulty as the full Verified set while requiring only 5GB of storage instead of 130GB. The model must autonomously explore, edit, and submit a patch through a multi-turn agent loop.\n\n## Task Description\n\n- **Task Type**: Automated Software Engineering / Bug Fixing (Agentic)\n- **Input**: GitHub issue description (no oracle file context)\n- **Output**: Code patch (diff format) collected from `git diff` after autonomous editing\n- **Size**: 50 samples (vs 500 in full Verified set)\n\n## Key Features\n\n- Representative 50-sample subset of SWE-bench Verified\n- Same difficulty distribution as the full dataset\n- Dramatically reduced storage requirements (5GB vs 130GB)\n- Multi-turn agent loop with per-instance Docker sandbox\n- Ideal for quick agentic evaluation and development iteration\n\n## Evaluation Notes\n\n- Requires `pip install swebench==4.1.0` before evaluation\n- Docker images are built/pulled automatically\n- See the [usage documentation](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench.html) for detailed setup\n- Good for rapid prototyping of agent strategies and initial model assessment\n\n## Agentic Mode\n\nThis benchmark drives a multi-turn agent loop (mirrors mini-swe-agent's\n`swebench.yaml`) inside a per-instance SWE-bench Docker container. The\nmodel issues `bash` commands to explore `/testbed`, edit source files,\nand finally submits its `git diff` patch by printing the sentinel\n`COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` followed by the patch contents.\n\nThe default `swe_bench_toolcall` strategy uses OpenAI function calling with\na single `bash` tool. Models without function-calling support can select\n`swe_bench_backticks` through `NativeAgentConfig.strategy`; that strategy\nexpects one ` ```mswea_bash_command ``` ` block per turn.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `swe_bench_verified_mini_agentic` |\n| **Dataset ID** | [evalscope/swe-bench-verified-mini](https://modelscope.cn/datasets/evalscope/swe-bench-verified-mini/summary) |\n| **Paper** | N/A |\n| **Tags** | `Coding` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 50 |\n| Prompt Length (Mean) | 1268.5 chars |\n| Prompt Length (Min/Max) | 257 / 5362 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"363bb967\",\n \"content\": \"AuthenticationForm's username field doesn't set maxlength HTML attribute.\\nDescription\\n\\t\\nAuthenticationForm's username field doesn't render with maxlength HTML attribute anymore.\\nRegression introduced in #27515 and 5ceaf14686ce626404afb6a5fbd3d8286410bf13.\\n​https://groups.google.com/forum/?utm_source=digest&utm_medium=email#!topic/django-developers/qnfSqro0DlA\\n​https://forum.djangoproject.com/t/possible-authenticationform-max-length-regression-in-django-2-1/241\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"problem_statement\": \"AuthenticationForm's username field doesn't set maxlength HTML attribute.\\nDescription\\n\\t\\nAuthenticationForm's username field doesn't render with maxlength HTML attribute anymore.\\nRegression introduced in #27515 and 5ceaf14686ce626404afb6a5fbd3d8286410bf13.\\n​https://groups.google.com/forum/?utm_source=digest&utm_medium=email#!topic/django-developers/qnfSqro0DlA\\n​https://forum.djangoproject.com/t/possible-authenticationform-max-length-regression-in-django-2-1/241\\n\",\n \"instance_id\": \"django__django-11790\",\n \"base_commit\": \"b1d6b35e146aea83b171c1b921178bbaae2795ed\",\n \"patch\": \"diff --git a/django/contrib/auth/forms.py b/django/contrib/auth/forms.py\\n--- a/django/contrib/auth/forms.py\\n+++ b/django/contrib/auth/forms.py\\n@@ -191,7 +191,9 @@ def __init__(self, request=None, *args, **kwargs):\\n \\n # Set the max len ... [TRUNCATED 322 chars] ... username_max_length\\n+ self.fields['username'].widget.attrs['maxlength'] = username_max_length\\n if self.fields['username'].label is None:\\n self.fields['username'].label = capfirst(self.username_field.verbose_name)\\n \\n\",\n \"PASS_TO_PASS\": [\n \"test_html_autocomplete_attributes (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_missing_passwords (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_non_matching_passwords (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_one_password (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_password_whitespace_not_stripped (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_success (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_field_order (auth_tests.test_forms.PasswordChangeFormTest)\",\n \"test_html_autocomplete_attributes (auth_tests.test_forms.PasswordChangeFormTest)\",\n \"test_incorrect_password (auth_tests.test_forms.PasswordChangeFormTest)\",\n \"test_password_verification (auth_tests.test_forms.PasswordChangeFormTest)\",\n \"... [TRUNCATED 67 more items] ...\"\n ],\n \"FAIL_TO_PASS\": [\n \"test_username_field_max_length_defaults_to_254 (auth_tests.test_forms.AuthenticationFormTest)\",\n \"test_username_field_max_length_matches_user_model (auth_tests.test_forms.AuthenticationFormTest)\"\n ],\n \"test_patch\": \"diff --git a/tests/auth_tests/test_forms.py b/tests/auth_tests/test_forms.py\\n--- a/tests/auth_tests/test_forms.py\\n+++ b/tests/auth_tests/test_forms.py\\n@@ -423,6 +423,7 @@ def test_username_field_max_length_matches_user_model(self):\\n C ... [TRUNCATED 543 chars] ... )\\n self.assertEqual(form.fields['username'].max_length, 254)\\n+ self.assertEqual(form.fields['username'].widget.attrs.get('maxlength'), 254)\\n self.assertEqual(form.errors, {})\\n \\n def test_username_field_label(self):\\n\",\n \"version\": \"3.1\",\n \"repo\": \"django/django\",\n \"environment_setup_commit\": \"0668164b4ac93a5be79f5b87fae83c657124d9ab\",\n \"hints_text\": \"Regression test.\",\n \"created_at\": \"2019-09-17T14:33:44Z\",\n \"docker_image\": \"swebench/sweb.eval.arm64.django_1776_django-11790:latest\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `build_docker_images` | `bool` | `True` | Build Docker images locally for each sample. |\n| `pull_remote_images_if_available` | `bool` | `True` | Attempt to pull existing remote Docker images before building. |\n| `force_arch` | `str` | `` | Optionally force a specific architecture for image build/pull. Choices: ['', 'arm64', 'x86_64'] |\n| `dockerhub_username` | `str` | `swebench` | DockerHub user/org namespace for remote SWE-bench images. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_verified_mini_agentic \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"swe_bench_toolcall\",\"max_steps\":250}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_verified_mini_agentic'],\n agent_config=NativeAgentConfig(\n strategy='swe_bench_toolcall',\n max_steps=250,\n ),\n dataset_args={\n 'swe_bench_verified_mini_agentic': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# SWE-bench_Verified_Mini_Agentic\n\n\n## 概述\n\nSWE-bench Verified Mini Agentic 是对 SWE-bench Verified Mini 的代理模式（agentic-mode）评估。SWE-bench Verified Mini 是一个精简的 50 样本子集，在保持与完整 Verified 集合相同性能分布、测试通过率和难度的同时，仅需 5GB 存储空间（而非 130GB）。模型必须通过多轮代理循环自主探索、编辑并提交补丁。\n\n## 任务描述\n\n- **任务类型**：自动化软件工程 / 缺陷修复（代理模式）\n- **输入**：GitHub issue 描述（无 oracle 文件上下文）\n- **输出**：代码补丁（diff 格式），通过 `git diff` 在自主编辑后收集\n- **规模**：50 个样本（完整 Verified 集合为 500 个）\n\n## 主要特性\n\n- SWE-bench Verified 的代表性 50 样本子集\n- 与完整数据集具有相同的难度分布\n- 存储需求大幅降低（5GB vs 130GB）\n- 每个实例使用独立 Docker 沙箱的多轮代理循环\n- 非常适合快速代理评估和开发迭代\n\n## 评估说明\n\n- 评估前需执行 `pip install swebench==4.1.0`\n- Docker 镜像会自动构建或拉取\n- 详细设置请参阅 [使用文档](https://evalscope.readthedocs.io/zh-cn/latest/third_party/swe_bench.html)\n- 适用于代理策略的快速原型设计和模型初步评估\n\n## 代理模式\n\n该基准测试在每个实例的 SWE-bench Docker 容器内驱动一个多轮代理循环（与 mini-swe-agent 的 `swebench.yaml` 配置一致）。模型通过发出 `bash` 命令来探索 `/testbed`、编辑源文件，并最终通过打印哨兵字符串 `COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT` 后跟补丁内容来提交其 `git diff` 补丁。\n\n默认的 `swe_bench_toolcall` 策略使用 OpenAI 函数调用，仅包含一个 `bash` 工具。不支持函数调用的模型可通过 `NativeAgentConfig.strategy` 选择 `swe_bench_backticks` 策略；该策略期望每轮输出一个 ` ```mswea_bash_command ``` ` 代码块。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `swe_bench_verified_mini_agentic` |\n| **数据集ID** | [evalscope/swe-bench-verified-mini](https://modelscope.cn/datasets/evalscope/swe-bench-verified-mini/summary) |\n| **论文** | N/A |\n| **标签** | `Coding` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估分割** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 50 |\n| 提示词长度（平均） | 1268.5 字符 |\n| 提示词长度（最小/最大） | 257 / 5362 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"363bb967\",\n \"content\": \"AuthenticationForm's username field doesn't set maxlength HTML attribute.\\nDescription\\n\\t\\nAuthenticationForm's username field doesn't render with maxlength HTML attribute anymore.\\nRegression introduced in #27515 and 5ceaf14686ce626404afb6a5fbd3d8286410bf13.\\n​https://groups.google.com/forum/?utm_source=digest&utm_medium=email#!topic/django-developers/qnfSqro0DlA\\n​https://forum.djangoproject.com/t/possible-authenticationform-max-length-regression-in-django-2-1/241\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"problem_statement\": \"AuthenticationForm's username field doesn't set maxlength HTML attribute.\\nDescription\\n\\t\\nAuthenticationForm's username field doesn't render with maxlength HTML attribute anymore.\\nRegression introduced in #27515 and 5ceaf14686ce626404afb6a5fbd3d8286410bf13.\\n​https://groups.google.com/forum/?utm_source=digest&utm_medium=email#!topic/django-developers/qnfSqro0DlA\\n​https://forum.djangoproject.com/t/possible-authenticationform-max-length-regression-in-django-2-1/241\\n\",\n \"instance_id\": \"django__django-11790\",\n \"base_commit\": \"b1d6b35e146aea83b171c1b921178bbaae2795ed\",\n \"patch\": \"diff --git a/django/contrib/auth/forms.py b/django/contrib/auth/forms.py\\n--- a/django/contrib/auth/forms.py\\n+++ b/django/contrib/auth/forms.py\\n@@ -191,7 +191,9 @@ def __init__(self, request=None, *args, **kwargs):\\n \\n # Set the max len ... [TRUNCATED 322 chars] ... username_max_length\\n+ self.fields['username'].widget.attrs['maxlength'] = username_max_length\\n if self.fields['username'].label is None:\\n self.fields['username'].label = capfirst(self.username_field.verbose_name)\\n \\n\",\n \"PASS_TO_PASS\": [\n \"test_html_autocomplete_attributes (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_missing_passwords (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_non_matching_passwords (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_one_password (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_password_whitespace_not_stripped (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_success (auth_tests.test_forms.AdminPasswordChangeFormTest)\",\n \"test_field_order (auth_tests.test_forms.PasswordChangeFormTest)\",\n \"test_html_autocomplete_attributes (auth_tests.test_forms.PasswordChangeFormTest)\",\n \"test_incorrect_password (auth_tests.test_forms.PasswordChangeFormTest)\",\n \"test_password_verification (auth_tests.test_forms.PasswordChangeFormTest)\",\n \"... [TRUNCATED 67 more items] ...\"\n ],\n \"FAIL_TO_PASS\": [\n \"test_username_field_max_length_defaults_to_254 (auth_tests.test_forms.AuthenticationFormTest)\",\n \"test_username_field_max_length_matches_user_model (auth_tests.test_forms.AuthenticationFormTest)\"\n ],\n \"test_patch\": \"diff --git a/tests/auth_tests/test_forms.py b/tests/auth_tests/test_forms.py\\n--- a/tests/auth_tests/test_forms.py\\n+++ b/tests/auth_tests/test_forms.py\\n@@ -423,6 +423,7 @@ def test_username_field_max_length_matches_user_model(self):\\n C ... [TRUNCATED 543 chars] ... )\\n self.assertEqual(form.fields['username'].max_length, 254)\\n+ self.assertEqual(form.fields['username'].widget.attrs.get('maxlength'), 254)\\n self.assertEqual(form.errors, {})\\n \\n def test_username_field_label(self):\\n\",\n \"version\": \"3.1\",\n \"repo\": \"django/django\",\n \"environment_setup_commit\": \"0668164b4ac93a5be79f5b87fae83c657124d9ab\",\n \"hints_text\": \"Regression test.\",\n \"created_at\": \"2019-09-17T14:33:44Z\",\n \"docker_image\": \"swebench/sweb.eval.arm64.django_1776_django-11790:latest\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板:**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `build_docker_images` | `bool` | `True` | 为每个样本在本地构建 Docker 镜像。 |\n| `pull_remote_images_if_available` | `bool` | `True` | 在构建前尝试拉取已存在的远程 Docker 镜像。 |\n| `force_arch` | `str` | `` | 可选地强制指定镜像构建/拉取的架构。选项：['', 'arm64', 'x86_64'] |\n| `dockerhub_username` | `str` | `swebench` | 远程 SWE-bench 镜像的 DockerHub 用户/组织命名空间。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets swe_bench_verified_mini_agentic \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"swe_bench_toolcall\",\"max_steps\":250}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['swe_bench_verified_mini_agentic'],\n agent_config=NativeAgentConfig(\n strategy='swe_bench_toolcall',\n max_steps=250,\n ),\n dataset_args={\n 'swe_bench_verified_mini_agentic': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "4375dbd2aca9e3386377cb07ef1ae48a", "needs_translation": false }, - "updated_at": "2026-06-16T10:56:20.668415", - "translation_updated_at": "2026-06-16T10:56:45" -} \ No newline at end of file + "updated_at": "2026-07-10T20:24:34.532023", + "translation_updated_at": "2026-07-10T20:24:42" +} diff --git a/evalscope/evalscope/benchmarks/_meta/tau2_bench.json b/evalscope/evalscope/benchmarks/_meta/tau2_bench.json index 49f4088..d950fad 100644 --- a/evalscope/evalscope/benchmarks/_meta/tau2_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/tau2_bench.json @@ -139,4 +139,4 @@ }, "updated_at": "2026-05-19T19:32:59.280180", "translation_updated_at": "2026-05-19T19:33:51" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/tau3_bench.json b/evalscope/evalscope/benchmarks/_meta/tau3_bench.json index a77054a..7269757 100644 --- a/evalscope/evalscope/benchmarks/_meta/tau3_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/tau3_bench.json @@ -164,4 +164,4 @@ }, "updated_at": "2026-05-19T19:45:52.676465", "translation_updated_at": "2026-05-19T19:45:55" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/tau_bench.json b/evalscope/evalscope/benchmarks/_meta/tau_bench.json index ddb82ab..1ebab0b 100644 --- a/evalscope/evalscope/benchmarks/_meta/tau_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/tau_bench.json @@ -69,4 +69,4 @@ }, "updated_at": "2026-01-28T17:31:38.723828", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/terminal_bench_v2.json b/evalscope/evalscope/benchmarks/_meta/terminal_bench_v2.json index 0c4c46c..5a3061f 100644 --- a/evalscope/evalscope/benchmarks/_meta/terminal_bench_v2.json +++ b/evalscope/evalscope/benchmarks/_meta/terminal_bench_v2.json @@ -87,4 +87,4 @@ }, "updated_at": "2026-05-27T18:11:56.412492", "translation_updated_at": "2026-05-27T18:11:58" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/terminal_bench_v2_1.json b/evalscope/evalscope/benchmarks/_meta/terminal_bench_v2_1.json index d50cfad..3e36ee1 100644 --- a/evalscope/evalscope/benchmarks/_meta/terminal_bench_v2_1.json +++ b/evalscope/evalscope/benchmarks/_meta/terminal_bench_v2_1.json @@ -87,4 +87,4 @@ }, "updated_at": "2026-05-27T18:11:56.412837", "translation_updated_at": "2026-05-27T18:11:58" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/tifa160.json b/evalscope/evalscope/benchmarks/_meta/tifa160.json index d7c4a87..39599ff 100644 --- a/evalscope/evalscope/benchmarks/_meta/tifa160.json +++ b/evalscope/evalscope/benchmarks/_meta/tifa160.json @@ -75,4 +75,4 @@ }, "updated_at": "2026-01-28T17:31:32.560824", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/tir_bench.json b/evalscope/evalscope/benchmarks/_meta/tir_bench.json index 60dfd17..055c101 100644 --- a/evalscope/evalscope/benchmarks/_meta/tir_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/tir_bench.json @@ -642,4 +642,4 @@ }, "updated_at": "2026-04-15T19:08:49.368638", "translation_updated_at": "2026-04-15T19:08:56Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/tool_bench.json b/evalscope/evalscope/benchmarks/_meta/tool_bench.json index 94416d1..ed377c2 100644 --- a/evalscope/evalscope/benchmarks/_meta/tool_bench.json +++ b/evalscope/evalscope/benchmarks/_meta/tool_bench.json @@ -142,4 +142,4 @@ }, "updated_at": "2026-05-15T14:45:08.636496", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/toolathlon.json b/evalscope/evalscope/benchmarks/_meta/toolathlon.json new file mode 100644 index 0000000..a999c60 --- /dev/null +++ b/evalscope/evalscope/benchmarks/_meta/toolathlon.json @@ -0,0 +1,177 @@ +{ + "meta": { + "pretty_name": "Toolathlon Official Service Wrapper", + "dataset_id": "https://github.com/hkust-nlp/Toolathlon", + "paper_url": null, + "tags": [ + "Agent", + "FunctionCalling", + "MultiTurn" + ], + "metrics": [ + "acc" + ], + "few_shot_num": 0, + "eval_split": "test", + "train_split": "", + "subset_list": [ + "default" + ], + "description": "\n## Overview\n\nToolathlon is an agent benchmark for realistic, long-horizon tool use across many MCP-backed software\nenvironments. This EvalScope benchmark is a wrapper around the official Toolathlon remote evaluation service,\nnot a local reimplementation of the MCP environments or official evaluator.\n\n## Evaluation Mode\n\n- Benchmark id: `toolathlon`\n- Supported mode: official service private mode\n- EvalScope controls model endpoint, task selection, job parameters, polling, result download, and reporting\n- The official Toolathlon service controls MCP environments, task containers, agent loop execution, and scoring\n- No Toolathlon, MCP application accounts, or Toolathlon Python package installation are required when using the\n official public evaluation service\n- A local or intranet OpenAI-compatible endpoint is required for private mode\n- EvalScope represents one remote Toolathlon job as one local sample; the generated data statistics count wrapper\n jobs, while `task_list` and `limit` control the Toolathlon tasks submitted inside that job\n- The bundled Toolathlon-Verified task list was inspected from official repository commit\n `b7bbac3f9a1f381b095c878debe1a47dd164ad85`\n\n## Usage Guide\n\nSee the Toolathlon usage guide for public-service limits, private-mode data flow, self-hosted service setup, and\nEvalScope configuration examples:\n\n- https://evalscope.readthedocs.io/en/latest/third_party/toolathlon.html\n\nOfficial sources:\n\n- https://github.com/hkust-nlp/Toolathlon\n- https://github.com/hkust-nlp/Toolathlon/blob/main/EVAL_SERVICE_README.md\n", + "prompt_template": "{question}", + "system_prompt": "", + "few_shot_prompt_template": "", + "aggregation": "mean", + "extra_params": { + "mode": { + "type": "str", + "description": "Toolathlon service mode. EvalScope supports private mode for this wrapper.", + "value": "private", + "choices": [ + "private" + ] + }, + "server_host": { + "type": "str", + "description": "Official Toolathlon evaluation service host.", + "value": "47.253.6.47" + }, + "server_port": { + "type": "int", + "description": "Official Toolathlon HTTP service port.", + "value": 8080 + }, + "ws_proxy_port": { + "type": "int", + "description": "Official Toolathlon WebSocket proxy port for private mode.", + "value": 8081 + }, + "workers": { + "type": "int", + "description": "Number of parallel Toolathlon workers requested from the official service.", + "value": 10 + }, + "provider": { + "type": "str", + "description": "Toolathlon model provider type.", + "value": "unified", + "choices": [ + "unified", + "openai_stateful_responses" + ] + }, + "task_list": { + "type": "list", + "description": "Optional Toolathlon task names to evaluate. Empty uses the bundled Toolathlon-Verified list.", + "value": [] + }, + "task_list_file": { + "type": "str", + "description": "Optional file containing one Toolathlon task name per line.", + "value": "" + }, + "model_params": { + "type": "dict", + "description": "Extra model parameters forwarded to Toolathlon, merged after TaskConfig.generation_config.", + "value": {} + }, + "job_id": { + "type": "str", + "description": "Optional Toolathlon job id. Reuse to resume an incomplete official-service job.", + "value": "" + }, + "force_redownload": { + "type": "bool", + "description": "Force redownload of Toolathlon result archives.", + "value": false + }, + "override_output_dir": { + "type": "bool", + "description": "Clear the Toolathlon output directory when it already contains files.", + "value": false + }, + "skip_container_restart": { + "type": "bool", + "description": "Skip Toolathlon container restart. Use only for small debug task subsets.", + "value": false + }, + "trust_env_in_httpx": { + "type": "bool", + "description": "Allow httpx to use proxy environment variables.", + "value": false + }, + "timeout_seconds": { + "type": "int", + "description": "Maximum time to wait for a Toolathlon official-service job.", + "value": 14400 + }, + "poll_interval": { + "type": "int", + "description": "Polling interval in seconds for Toolathlon job status.", + "value": 5 + } + }, + "sandbox_config": {}, + "category": "agent" + }, + "statistics": { + "total_samples": 1, + "subset_stats": [ + { + "name": "default", + "sample_count": 1, + "prompt_length_mean": 50, + "prompt_length_min": 50, + "prompt_length_max": 50, + "prompt_length_std": null, + "target_length_mean": null + } + ], + "prompt_length": { + "mean": 50, + "min": 50, + "max": 50, + "std": null + }, + "target_length_mean": null, + "computed_at": "2026-07-10T10:05:26.379976" + }, + "sample_example": { + "data": { + "input": [ + { + "id": "22a512d8", + "content": "Run Toolathlon official remote evaluation service." + } + ], + "target": "", + "id": 0, + "metadata": { + "task_list": [ + "ab-testing", + "academic-pdf-report", + "academic-warning", + "add-bibtex", + "apply-phd-email", + "arrange-workspace", + "canvas-arrange-exam", + "canvas-art-manager", + "canvas-art-quiz", + "canvas-do-quiz", + "... [TRUNCATED 98 more items] ..." + ], + "mode": "private" + } + }, + "subset": "default", + "truncated": false + }, + "readme": { + "en": "# Toolathlon Official Service Wrapper\n\n\n## Overview\n\nToolathlon is an agent benchmark for realistic, long-horizon tool use across many MCP-backed software\nenvironments. This EvalScope benchmark is a wrapper around the official Toolathlon remote evaluation service,\nnot a local reimplementation of the MCP environments or official evaluator.\n\n## Evaluation Mode\n\n- Benchmark id: `toolathlon`\n- Supported mode: official service private mode\n- EvalScope controls model endpoint, task selection, job parameters, polling, result download, and reporting\n- The official Toolathlon service controls MCP environments, task containers, agent loop execution, and scoring\n- No Toolathlon, MCP application accounts, or Toolathlon Python package installation are required when using the\n official public evaluation service\n- A local or intranet OpenAI-compatible endpoint is required for private mode\n- EvalScope represents one remote Toolathlon job as one local sample; the generated data statistics count wrapper\n jobs, while `task_list` and `limit` control the Toolathlon tasks submitted inside that job\n- The bundled Toolathlon-Verified task list was inspected from official repository commit\n `b7bbac3f9a1f381b095c878debe1a47dd164ad85`\n\n## Usage Guide\n\nSee the Toolathlon usage guide for public-service limits, private-mode data flow, self-hosted service setup, and\nEvalScope configuration examples:\n\n- https://evalscope.readthedocs.io/en/latest/third_party/toolathlon.html\n\nOfficial sources:\n\n- https://github.com/hkust-nlp/Toolathlon\n- https://github.com/hkust-nlp/Toolathlon/blob/main/EVAL_SERVICE_README.md\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `toolathlon` |\n| **Dataset ID** | [Toolathlon](https://github.com/hkust-nlp/Toolathlon) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `FunctionCalling`, `MultiTurn` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 1 |\n| Prompt Length (Mean) | 50 chars |\n| Prompt Length (Min/Max) | 50 / 50 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"22a512d8\",\n \"content\": \"Run Toolathlon official remote evaluation service.\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"metadata\": {\n \"task_list\": [\n \"ab-testing\",\n \"academic-pdf-report\",\n \"academic-warning\",\n \"add-bibtex\",\n \"apply-phd-email\",\n \"arrange-workspace\",\n \"canvas-arrange-exam\",\n \"canvas-art-manager\",\n \"canvas-art-quiz\",\n \"canvas-do-quiz\",\n \"... [TRUNCATED 98 more items] ...\"\n ],\n \"mode\": \"private\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `mode` | `str` | `private` | Toolathlon service mode. EvalScope supports private mode for this wrapper. Choices: ['private'] |\n| `server_host` | `str` | `47.253.6.47` | Official Toolathlon evaluation service host. |\n| `server_port` | `int` | `8080` | Official Toolathlon HTTP service port. |\n| `ws_proxy_port` | `int` | `8081` | Official Toolathlon WebSocket proxy port for private mode. |\n| `workers` | `int` | `10` | Number of parallel Toolathlon workers requested from the official service. |\n| `provider` | `str` | `unified` | Toolathlon model provider type. Choices: ['unified', 'openai_stateful_responses'] |\n| `task_list` | `list` | `[]` | Optional Toolathlon task names to evaluate. Empty uses the bundled Toolathlon-Verified list. |\n| `task_list_file` | `str` | `` | Optional file containing one Toolathlon task name per line. |\n| `model_params` | `dict` | `{}` | Extra model parameters forwarded to Toolathlon, merged after TaskConfig.generation_config. |\n| `job_id` | `str` | `` | Optional Toolathlon job id. Reuse to resume an incomplete official-service job. |\n| `force_redownload` | `bool` | `False` | Force redownload of Toolathlon result archives. |\n| `override_output_dir` | `bool` | `False` | Clear the Toolathlon output directory when it already contains files. |\n| `skip_container_restart` | `bool` | `False` | Skip Toolathlon container restart. Use only for small debug task subsets. |\n| `trust_env_in_httpx` | `bool` | `False` | Allow httpx to use proxy environment variables. |\n| `timeout_seconds` | `int` | `14400` | Maximum time to wait for a Toolathlon official-service job. |\n| `poll_interval` | `int` | `5` | Polling interval in seconds for Toolathlon job status. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets toolathlon \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['toolathlon'],\n dataset_args={\n 'toolathlon': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# Toolathlon Official Service Wrapper\n\n\n## 概述\n\nToolathlon 是一个面向现实场景、长周期工具使用的智能体基准测试，覆盖多种基于 MCP 的软件环境。本 EvalScope 基准测试是对官方 Toolathlon 远程评估服务的封装，并非对 MCP 环境或官方评估器的本地重新实现。\n\n## 评估模式\n\n- 基准测试 ID：`toolathlon`\n- 支持模式：官方服务私有模式（official service private mode）\n- EvalScope 负责控制模型端点、任务选择、作业参数、轮询、结果下载和报告生成\n- 官方 Toolathlon 服务负责控制 MCP 环境、任务容器、智能体循环执行和评分\n- 使用官方公共评估服务时，无需安装 Toolathlon、MCP 应用账户或 Toolathlon Python 包\n- 私有模式下需要一个本地或内网的 OpenAI 兼容端点\n- EvalScope 将一个远程 Toolathlon 作业视为一个本地样本；生成的数据统计计数的是封装作业数量，而 `task_list` 和 `limit` 控制该作业内部提交的 Toolathlon 任务\n- 所附带的 Toolathlon-Verified 任务列表源自官方仓库提交记录 `b7bbac3f9a1f381b095c878debe1a47dd164ad85`\n\n## 使用指南\n\n有关公共服务限制、私有模式数据流、自托管服务设置以及 EvalScope 配置示例，请参阅 Toolathlon 使用指南：\n\n- https://evalscope.readthedocs.io/zh-cn/latest/third_party/toolathlon.html\n\n官方资源：\n\n- https://github.com/hkust-nlp/Toolathlon\n- https://github.com/hkust-nlp/Toolathlon/blob/main/EVAL_SERVICE_README.md\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `toolathlon` |\n| **数据集ID** | [Toolathlon](https://github.com/hkust-nlp/Toolathlon) |\n| **论文** | N/A |\n| **标签** | `Agent`, `FunctionCalling`, `MultiTurn` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 1 |\n| 提示词长度（平均） | 50 字符 |\n| 提示词长度（最小/最大） | 50 / 50 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"22a512d8\",\n \"content\": \"Run Toolathlon official remote evaluation service.\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"metadata\": {\n \"task_list\": [\n \"ab-testing\",\n \"academic-pdf-report\",\n \"academic-warning\",\n \"add-bibtex\",\n \"apply-phd-email\",\n \"arrange-workspace\",\n \"canvas-arrange-exam\",\n \"canvas-art-manager\",\n \"canvas-art-quiz\",\n \"canvas-do-quiz\",\n \"... [TRUNCATED 98 more items] ...\"\n ],\n \"mode\": \"private\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板:**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `mode` | `str` | `private` | Toolathlon 服务模式。EvalScope 此封装仅支持私有模式。选项：['private'] |\n| `server_host` | `str` | `47.253.6.47` | 官方 Toolathlon 评估服务主机地址。 |\n| `server_port` | `int` | `8080` | 官方 Toolathlon HTTP 服务端口。 |\n| `ws_proxy_port` | `int` | `8081` | 私有模式下官方 Toolathlon WebSocket 代理端口。 |\n| `workers` | `int` | `10` | 向官方服务请求的并行 Toolathlon 工作进程数量。 |\n| `provider` | `str` | `unified` | Toolathlon 模型提供者类型。选项：['unified', 'openai_stateful_responses'] |\n| `task_list` | `list` | `[]` | 可选的待评估 Toolathlon 任务名称列表。留空则使用内置的 Toolathlon-Verified 列表。 |\n| `task_list_file` | `str` | `` | 可选文件路径，每行包含一个 Toolathlon 任务名称。 |\n| `model_params` | `dict` | `{}` | 额外模型参数，将转发给 Toolathlon，并在 TaskConfig.generation_config 之后合并。 |\n| `job_id` | `str` | `` | 可选的 Toolathlon 作业 ID。可用于恢复未完成的官方服务作业。 |\n| `force_redownload` | `bool` | `False` | 强制重新下载 Toolathlon 结果压缩包。 |\n| `override_output_dir` | `bool` | `False` | 当 Toolathlon 输出目录已存在文件时，是否清空目录。 |\n| `skip_container_restart` | `bool` | `False` | 跳过 Toolathlon 容器重启。仅用于小型调试任务子集。 |\n| `trust_env_in_httpx` | `bool` | `False` | 允许 httpx 使用代理环境变量。 |\n| `timeout_seconds` | `int` | `14400` | 等待 Toolathlon 官方服务作业的最大时间（秒）。 |\n| `poll_interval` | `int` | `5` | 轮询 Toolathlon 作业状态的时间间隔（秒）。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets toolathlon \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['toolathlon'],\n dataset_args={\n 'toolathlon': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "1bd327d70c75f087d1902656a5ddb66e", + "needs_translation": false + }, + "updated_at": "2026-07-10T10:05:26.380342", + "translation_updated_at": "2026-07-10T10:05:29" +} diff --git a/evalscope/evalscope/benchmarks/_meta/torgo.json b/evalscope/evalscope/benchmarks/_meta/torgo.json index 4ec2763..585d9c3 100644 --- a/evalscope/evalscope/benchmarks/_meta/torgo.json +++ b/evalscope/evalscope/benchmarks/_meta/torgo.json @@ -178,4 +178,4 @@ }, "updated_at": "2026-01-28T17:31:32.629384", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/trivia_qa.json b/evalscope/evalscope/benchmarks/_meta/trivia_qa.json index 01aa334..fe0ac91 100644 --- a/evalscope/evalscope/benchmarks/_meta/trivia_qa.json +++ b/evalscope/evalscope/benchmarks/_meta/trivia_qa.json @@ -21,7 +21,7 @@ "rc.wikipedia" ], "description": "\n## Overview\n\nTriviaQA is a large-scale reading comprehension dataset containing over 650K question-answer-evidence triples. Questions are collected from trivia enthusiast websites and paired with Wikipedia articles as evidence documents.\n\n## Task Description\n\n- **Task Type**: Reading Comprehension / Question Answering\n- **Input**: Question with Wikipedia context passage\n- **Output**: Answer extracted or generated from context\n- **Domain**: General knowledge trivia questions\n\n## Key Features\n\n- 650K+ question-answer-evidence triples\n- Questions written by trivia enthusiasts (naturally challenging)\n- Multiple valid answer aliases for flexible evaluation\n- Wikipedia articles provide evidence passages\n- Tests both reading comprehension and knowledge retrieval\n\n## Evaluation Notes\n\n- Default configuration uses **0-shot** evaluation\n- Uses the Wikipedia reading comprehension subset (rc.wikipedia)\n- Answers should follow the format: \"ANSWER: [ANSWER]\"\n- Supports inclusion-based matching for answer comparison\n- Evaluates on validation split\n", - "prompt_template": "Read the content and answer the following question.\n\nContent: {content}\n\nQuestion: {question}\n\nKeep your The last line of your response should be of the form \"ANSWER: [ANSWER]\" (without quotes) where [ANSWER] is the answer to the problem.\n", + "prompt_template": "Read the content and answer the following question.\n\nContent: {content}\n\nQuestion: {question}\n\nThe last line of your response should be of the form \"ANSWER: [ANSWER]\" (without quotes) where [ANSWER] is the answer to the problem.\n", "system_prompt": "", "few_shot_prompt_template": "", "aggregation": "mean", @@ -35,28 +35,28 @@ { "name": "rc.wikipedia", "sample_count": 7993, - "prompt_length_mean": 54126.56, - "prompt_length_min": 339, - "prompt_length_max": 691325, - "prompt_length_std": 51613.67, - "target_length_mean": 463.42 + "prompt_length_mean": 53522.95, + "prompt_length_min": 329, + "prompt_length_max": 691315, + "prompt_length_std": 51017.99, + "target_length_mean": 443.85 } ], "prompt_length": { - "mean": 54126.56, - "min": 339, - "max": 691325, - "std": 51613.67 + "mean": 53522.95, + "min": 329, + "max": 691315, + "std": 51017.99 }, - "target_length_mean": 463.42, - "computed_at": "2026-01-28T11:17:13.009872" + "target_length_mean": 443.85, + "computed_at": "2026-07-10T10:44:35.689061" }, "sample_example": { "data": { "input": [ { - "id": "545e4eda", - "content": "Read the content and answer the following question.\n\nContent: ['Andrew Lloyd Webber, Baron Lloyd-Webber (born 22 March 1948) is an English composer and impresario of musical theatre. \\n\\nSeveral of his musicals have run for more than a deca ... [TRUNCATED] ... ening titles.']\n\nQuestion: Which Lloyd Webber musical premiered in the US on 10th December 1993?\n\nKeep your The last line of your response should be of the form \"ANSWER: [ANSWER]\" (without quotes) where [ANSWER] is the answer to the problem.\n" + "id": "3813a170", + "content": "Read the content and answer the following question.\n\nContent: ['Andrew Lloyd Webber, Baron Lloyd-Webber (born 22 March 1948) is an English composer and impresario of musical theatre. \\n\\nSeveral of his musicals have run for more than a deca ... [TRUNCATED 32057 chars] ... show\\'s opening titles.']\n\nQuestion: Which Lloyd Webber musical premiered in the US on 10th December 1993?\n\nThe last line of your response should be of the form \"ANSWER: [ANSWER]\" (without quotes) where [ANSWER] is the answer to the problem.\n" } ], "target": [ @@ -75,7 +75,7 @@ "metadata": { "question_id": "tc_33", "content": [ - "Andrew Lloyd Webber, Baron Lloyd-Webber (born 22 March 1948) is an English composer and impresario of musical theatre. \n\nSeveral of his musicals have run for more than a decade both in the West End and on Broadway. He has composed 13 musica ... [TRUNCATED] ... same name, composed the song \"Fields of Sun\". The actual song was never used on the show, nor was it available on the CD soundtrack that was released at the time. He was however still credited for the unused song in the show's opening titles." + "Andrew Lloyd Webber, Baron Lloyd-Webber (born 22 March 1948) is an English composer and impresario of musical theatre. \n\nSeveral of his musicals have run for more than a decade both in the West End and on Broadway. He has composed 13 musica ... [TRUNCATED 31402 chars] ... same name, composed the song \"Fields of Sun\". The actual song was never used on the show, nor was it available on the CD soundtrack that was released at the time. He was however still credited for the unused song in the show's opening titles." ] } }, @@ -83,11 +83,11 @@ "truncated": true }, "readme": { - "en": "# TriviaQA\n\n\n## Overview\n\nTriviaQA is a large-scale reading comprehension dataset containing over 650K question-answer-evidence triples. Questions are collected from trivia enthusiast websites and paired with Wikipedia articles as evidence documents.\n\n## Task Description\n\n- **Task Type**: Reading Comprehension / Question Answering\n- **Input**: Question with Wikipedia context passage\n- **Output**: Answer extracted or generated from context\n- **Domain**: General knowledge trivia questions\n\n## Key Features\n\n- 650K+ question-answer-evidence triples\n- Questions written by trivia enthusiasts (naturally challenging)\n- Multiple valid answer aliases for flexible evaluation\n- Wikipedia articles provide evidence passages\n- Tests both reading comprehension and knowledge retrieval\n\n## Evaluation Notes\n\n- Default configuration uses **0-shot** evaluation\n- Uses the Wikipedia reading comprehension subset (rc.wikipedia)\n- Answers should follow the format: \"ANSWER: [ANSWER]\"\n- Supports inclusion-based matching for answer comparison\n- Evaluates on validation split\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `trivia_qa` |\n| **Dataset ID** | [evalscope/trivia_qa](https://modelscope.cn/datasets/evalscope/trivia_qa/summary) |\n| **Paper** | N/A |\n| **Tags** | `QA`, `ReadingComprehension` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `validation` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 7,993 |\n| Prompt Length (Mean) | 54126.56 chars |\n| Prompt Length (Min/Max) | 339 / 691325 chars |\n\n## Sample Example\n\n**Subset**: `rc.wikipedia`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"545e4eda\",\n \"content\": \"Read the content and answer the following question.\\n\\nContent: ['Andrew Lloyd Webber, Baron Lloyd-Webber (born 22 March 1948) is an English composer and impresario of musical theatre. \\\\n\\\\nSeveral of his musicals have run for more than a deca ... [TRUNCATED] ... ening titles.']\\n\\nQuestion: Which Lloyd Webber musical premiered in the US on 10th December 1993?\\n\\nKeep your The last line of your response should be of the form \\\"ANSWER: [ANSWER]\\\" (without quotes) where [ANSWER] is the answer to the problem.\\n\"\n }\n ],\n \"target\": [\n \"Sunset Blvd\",\n \"West Sunset Boulevard\",\n \"Sunset Boulevard\",\n \"Sunset Bulevard\",\n \"Sunset Blvd.\",\n \"sunset boulevard\",\n \"sunset bulevard\",\n \"west sunset boulevard\",\n \"sunset blvd\"\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"question_id\": \"tc_33\",\n \"content\": [\n \"Andrew Lloyd Webber, Baron Lloyd-Webber (born 22 March 1948) is an English composer and impresario of musical theatre. \\n\\nSeveral of his musicals have run for more than a decade both in the West End and on Broadway. He has composed 13 musica ... [TRUNCATED] ... same name, composed the song \\\"Fields of Sun\\\". The actual song was never used on the show, nor was it available on the CD soundtrack that was released at the time. He was however still credited for the unused song in the show's opening titles.\"\n ]\n }\n}\n```\n\n*Note: Some content was truncated for display.*\n\n## Prompt Template\n\n**Prompt Template:**\n```text\nRead the content and answer the following question.\n\nContent: {content}\n\nQuestion: {question}\n\nKeep your The last line of your response should be of the form \"ANSWER: [ANSWER]\" (without quotes) where [ANSWER] is the answer to the problem.\n\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets trivia_qa \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['trivia_qa'],\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# TriviaQA\n\n\n## 概述\n\nTriviaQA 是一个大规模阅读理解数据集，包含超过 65 万个问题-答案-证据三元组。问题收集自问答爱好者网站，并与维基百科文章配对作为证据文档。\n\n## 任务描述\n\n- **任务类型**：阅读理解 / 问答\n- **输入**：带有维基百科上下文段落的问题\n- **输出**：从上下文中抽取或生成的答案\n- **领域**：通用知识类问答\n\n## 主要特点\n\n- 超过 65 万个问题-答案-证据三元组\n- 问题由问答爱好者编写（天然具有挑战性）\n- 支持多个有效答案别名，便于灵活评估\n- 维基百科文章提供证据段落\n- 同时考察阅读理解与知识检索能力\n\n## 评估说明\n\n- 默认配置使用 **0-shot** 评估\n- 使用维基百科阅读理解子集（rc.wikipedia）\n- 答案格式应为：\"ANSWER: [ANSWER]\"\n- 支持基于包含关系的答案匹配\n- 在验证集（validation split）上进行评估\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `trivia_qa` |\n| **数据集ID** | [evalscope/trivia_qa](https://modelscope.cn/datasets/evalscope/trivia_qa/summary) |\n| **论文** | N/A |\n| **标签** | `QA`, `ReadingComprehension` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `validation` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 7,993 |\n| 提示词长度（平均） | 54126.56 字符 |\n| 提示词长度（最小/最大） | 339 / 691325 字符 |\n\n## 样例示例\n\n**子集**: `rc.wikipedia`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"545e4eda\",\n \"content\": \"Read the content and answer the following question.\\n\\nContent: ['Andrew Lloyd Webber, Baron Lloyd-Webber (born 22 March 1948) is an English composer and impresario of musical theatre. \\\\n\\\\nSeveral of his musicals have run for more than a deca ... [TRUNCATED] ... ening titles.']\\n\\nQuestion: Which Lloyd Webber musical premiered in the US on 10th December 1993?\\n\\nKeep your The last line of your response should be of the form \\\"ANSWER: [ANSWER]\\\" (without quotes) where [ANSWER] is the answer to the problem.\\n\"\n }\n ],\n \"target\": [\n \"Sunset Blvd\",\n \"West Sunset Boulevard\",\n \"Sunset Boulevard\",\n \"Sunset Bulevard\",\n \"Sunset Blvd.\",\n \"sunset boulevard\",\n \"sunset bulevard\",\n \"west sunset boulevard\",\n \"sunset blvd\"\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"question_id\": \"tc_33\",\n \"content\": [\n \"Andrew Lloyd Webber, Baron Lloyd-Webber (born 22 March 1948) is an English composer and impresario of musical theatre. \\n\\nSeveral of his musicals have run for more than a decade both in the West End and on Broadway. He has composed 13 musica ... [TRUNCATED] ... same name, composed the song \\\"Fields of Sun\\\". The actual song was never used on the show, nor was it available on the CD soundtrack that was released at the time. He was however still credited for the unused song in the show's opening titles.\"\n ]\n }\n}\n```\n\n*注：部分内容因展示需要已被截断。*\n\n## 提示模板\n\n**提示模板：**\n```text\nRead the content and answer the following question.\n\nContent: {content}\n\nQuestion: {question}\n\nKeep your The last line of your response should be of the form \"ANSWER: [ANSWER]\" (without quotes) where [ANSWER] is the answer to the problem.\n\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets trivia_qa \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['trivia_qa'],\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "9781e6776a082c49fe8e168426c315a6", + "en": "# TriviaQA\n\n\n## Overview\n\nTriviaQA is a large-scale reading comprehension dataset containing over 650K question-answer-evidence triples. Questions are collected from trivia enthusiast websites and paired with Wikipedia articles as evidence documents.\n\n## Task Description\n\n- **Task Type**: Reading Comprehension / Question Answering\n- **Input**: Question with Wikipedia context passage\n- **Output**: Answer extracted or generated from context\n- **Domain**: General knowledge trivia questions\n\n## Key Features\n\n- 650K+ question-answer-evidence triples\n- Questions written by trivia enthusiasts (naturally challenging)\n- Multiple valid answer aliases for flexible evaluation\n- Wikipedia articles provide evidence passages\n- Tests both reading comprehension and knowledge retrieval\n\n## Evaluation Notes\n\n- Default configuration uses **0-shot** evaluation\n- Uses the Wikipedia reading comprehension subset (rc.wikipedia)\n- Answers should follow the format: \"ANSWER: [ANSWER]\"\n- Supports inclusion-based matching for answer comparison\n- Evaluates on validation split\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `trivia_qa` |\n| **Dataset ID** | [evalscope/trivia_qa](https://modelscope.cn/datasets/evalscope/trivia_qa/summary) |\n| **Paper** | N/A |\n| **Tags** | `QA`, `ReadingComprehension` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `validation` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 7,993 |\n| Prompt Length (Mean) | 53522.95 chars |\n| Prompt Length (Min/Max) | 329 / 691315 chars |\n\n## Sample Example\n\n**Subset**: `rc.wikipedia`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"3813a170\",\n \"content\": \"Read the content and answer the following question.\\n\\nContent: ['Andrew Lloyd Webber, Baron Lloyd-Webber (born 22 March 1948) is an English composer and impresario of musical theatre. \\\\n\\\\nSeveral of his musicals have run for more than a deca ... [TRUNCATED 32057 chars] ... show\\\\'s opening titles.']\\n\\nQuestion: Which Lloyd Webber musical premiered in the US on 10th December 1993?\\n\\nThe last line of your response should be of the form \\\"ANSWER: [ANSWER]\\\" (without quotes) where [ANSWER] is the answer to the problem.\\n\"\n }\n ],\n \"target\": [\n \"Sunset Blvd\",\n \"West Sunset Boulevard\",\n \"Sunset Boulevard\",\n \"Sunset Bulevard\",\n \"Sunset Blvd.\",\n \"sunset boulevard\",\n \"sunset bulevard\",\n \"west sunset boulevard\",\n \"sunset blvd\"\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"question_id\": \"tc_33\",\n \"content\": [\n \"Andrew Lloyd Webber, Baron Lloyd-Webber (born 22 March 1948) is an English composer and impresario of musical theatre. \\n\\nSeveral of his musicals have run for more than a decade both in the West End and on Broadway. He has composed 13 musica ... [TRUNCATED 31402 chars] ... same name, composed the song \\\"Fields of Sun\\\". The actual song was never used on the show, nor was it available on the CD soundtrack that was released at the time. He was however still credited for the unused song in the show's opening titles.\"\n ]\n }\n}\n```\n\n*Note: Some content was truncated for display.*\n\n## Prompt Template\n\n**Prompt Template:**\n```text\nRead the content and answer the following question.\n\nContent: {content}\n\nQuestion: {question}\n\nThe last line of your response should be of the form \"ANSWER: [ANSWER]\" (without quotes) where [ANSWER] is the answer to the problem.\n\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets trivia_qa \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['trivia_qa'],\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# TriviaQA\n\n\n## 概述\n\nTriviaQA 是一个大规模阅读理解数据集，包含超过 65 万个问题-答案-证据三元组。问题收集自问答爱好者网站，并配以维基百科文章作为证据文档。\n\n## 任务描述\n\n- **任务类型**：阅读理解 / 问答\n- **输入**：带有维基百科上下文段落的问题\n- **输出**：从上下文中抽取或生成的答案\n- **领域**：通用知识类问答\n\n## 主要特点\n\n- 包含 65 万+ 个问题-答案-证据三元组\n- 问题由问答爱好者编写（天然具有挑战性）\n- 支持多个有效答案别名，便于灵活评估\n- 维基百科文章提供证据段落\n- 同时考察阅读理解与知识检索能力\n\n## 评估说明\n\n- 默认配置使用 **0-shot** 评估\n- 使用维基百科阅读理解子集（rc.wikipedia）\n- 答案格式应为：\"ANSWER: [ANSWER]\"\n- 支持基于包含关系的答案匹配方式\n- 在验证集（validation split）上进行评估\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `trivia_qa` |\n| **数据集ID** | [evalscope/trivia_qa](https://modelscope.cn/datasets/evalscope/trivia_qa/summary) |\n| **论文** | N/A |\n| **标签** | `QA`, `ReadingComprehension` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `validation` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 7,993 |\n| 提示词长度（平均） | 53522.95 字符 |\n| 提示词长度（最小/最大） | 329 / 691315 字符 |\n\n## 样例示例\n\n**子集**: `rc.wikipedia`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"3813a170\",\n \"content\": \"Read the content and answer the following question.\\n\\nContent: ['Andrew Lloyd Webber, Baron Lloyd-Webber (born 22 March 1948) is an English composer and impresario of musical theatre. \\\\n\\\\nSeveral of his musicals have run for more than a deca ... [TRUNCATED 32057 chars] ... show\\\\'s opening titles.']\\n\\nQuestion: Which Lloyd Webber musical premiered in the US on 10th December 1993?\\n\\nThe last line of your response should be of the form \\\"ANSWER: [ANSWER]\\\" (without quotes) where [ANSWER] is the answer to the problem.\\n\"\n }\n ],\n \"target\": [\n \"Sunset Blvd\",\n \"West Sunset Boulevard\",\n \"Sunset Boulevard\",\n \"Sunset Bulevard\",\n \"Sunset Blvd.\",\n \"sunset boulevard\",\n \"sunset bulevard\",\n \"west sunset boulevard\",\n \"sunset blvd\"\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"question_id\": \"tc_33\",\n \"content\": [\n \"Andrew Lloyd Webber, Baron Lloyd-Webber (born 22 March 1948) is an English composer and impresario of musical theatre. \\n\\nSeveral of his musicals have run for more than a decade both in the West End and on Broadway. He has composed 13 musica ... [TRUNCATED 31402 chars] ... same name, composed the song \\\"Fields of Sun\\\". The actual song was never used on the show, nor was it available on the CD soundtrack that was released at the time. He was however still credited for the unused song in the show's opening titles.\"\n ]\n }\n}\n```\n\n*注：部分内容因展示需要已被截断。*\n\n## 提示模板\n\n**提示模板：**\n```text\nRead the content and answer the following question.\n\nContent: {content}\n\nQuestion: {question}\n\nThe last line of your response should be of the form \"ANSWER: [ANSWER]\" (without quotes) where [ANSWER] is the answer to the problem.\n\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets trivia_qa \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['trivia_qa'],\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "4ac3b3159577886a8a188bedfa76b982", "needs_translation": false }, - "updated_at": "2026-01-28T17:31:32.638498", - "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file + "updated_at": "2026-07-10T11:50:16.342667", + "translation_updated_at": "2026-07-10T11:50:40" +} diff --git a/evalscope/evalscope/benchmarks/_meta/truthful_qa.json b/evalscope/evalscope/benchmarks/_meta/truthful_qa.json index 3d271c4..6c56f23 100644 --- a/evalscope/evalscope/benchmarks/_meta/truthful_qa.json +++ b/evalscope/evalscope/benchmarks/_meta/truthful_qa.json @@ -86,4 +86,4 @@ }, "updated_at": "2026-01-28T17:31:32.646477", "translation_updated_at": "2026-01-28T16:09:53Z" -} \ No newline at end of file +} diff --git a/evalscope/evalscope/benchmarks/_meta/tvbench.json b/evalscope/evalscope/benchmarks/_meta/tvbench.json new file mode 100644 index 0000000..7146603 --- /dev/null +++ b/evalscope/evalscope/benchmarks/_meta/tvbench.json @@ -0,0 +1,144 @@ +{ + "meta": { + "pretty_name": "TVBench", + "dataset_id": "evalscope/TVBench", + "paper_url": null, + "tags": [ + "MultiModal", + "Video", + "MCQ" + ], + "metrics": [ + "acc" + ], + "few_shot_num": 0, + "eval_split": "train", + "train_split": "", + "subset_list": [ + "action_count" + ], + "description": "\n## Overview\n\nTVBench is a temporal video understanding benchmark for evaluating whether multimodal models can reason over dynamic visual events rather than isolated frames. It covers a broad set of video reasoning skills, including action recognition, action counting, temporal localization, action order understanding, egocentric action sequencing, object counting, object shuffling, moving direction recognition, scene transition reasoning, and unexpected action detection.\n\nThe EvalScope native adapter reads the official per-task JSON annotations from the dataset repository and resolves the corresponding video archives on demand. This keeps the default smoke-test path lightweight while still supporting the full benchmark through `subset_list`.\n\n## Task Description\n\n- **Task Type**: Video multiple-choice question answering (MCQ)\n- **Input**: A video clip or a time-bounded video segment, a natural-language question, and 2-4 answer candidates\n- **Output**: A single answer option letter selected from the provided candidates\n- **Default Subset**: `action_count`, selected because it is available as a standard MP4 archive in the public dataset repository\n- **Supported Subsets**: `action_antonym`, `action_count`, `action_localization`, `action_sequence`, `egocentric_sequence`, `moving_direction`, `object_count`, `object_shuffle`, `scene_transition`, and `unexpected_action`\n\n## Evaluation Notes\n\n- Default evaluation uses 0-shot Chain-of-Thought multiple-choice prompting via `MultipleChoiceTemplate.SINGLE_ANSWER_COT`.\n- Primary metric: Accuracy (`acc`). The dataset answer is stored as candidate text and is converted to the corresponding option letter before scoring.\n- Some subsets provide `start`/`end` fields. The adapter passes these values to `ContentVideo` and also adds a concise segment instruction to the prompt.\n- Video files are downloaded lazily from subset-specific archives. `egocentric_sequence` uses segmented archives under `video/egocentric_sequence/.zip`.\n- The `action_antonym` annotations reference AVI files. If the repository media archive is unavailable, configure `extra_params.video_dir` to a local directory containing the AVI files.\n- The adapter supports local video layouts that are either flat (`video_dir/