13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
202 lines
22 KiB
JSON
202 lines
22 KiB
JSON
{
|
||
"meta": {
|
||
"pretty_name": "WideSearch",
|
||
"dataset_id": "bytedance-community/WideSearch",
|
||
"paper_url": "https://arxiv.org/abs/2508.07999",
|
||
"tags": [
|
||
"Agent",
|
||
"MultiTurn",
|
||
"Retrieval"
|
||
],
|
||
"metrics": [
|
||
"success_rate",
|
||
"row_precision",
|
||
"row_recall",
|
||
"row_f1",
|
||
"item_precision",
|
||
"item_recall",
|
||
"item_f1"
|
||
],
|
||
"few_shot_num": 0,
|
||
"eval_split": "full",
|
||
"train_split": "",
|
||
"subset_list": [
|
||
"default"
|
||
],
|
||
"description": "\n## Overview\n\nWideSearch evaluates search agents on broad web information-seeking tasks. Each task asks the agent to collect many\natomic facts and return one structured Markdown table. EvalScope uses the ModelScope\n`bytedance-community/WideSearch` dataset.\n\n## Task Description\n\n- **Task Type**: Multi-turn search agent\n- **Input**: Natural-language collection request with an explicit table schema\n- **Output**: Complete Markdown table\n- **Dataset**: 200 tasks in the ``full`` split; 100 English and 100 Chinese\n\n## Key Features\n\n- Official single-agent protocol: language-specific system prompt, ``function_calling``, and 50 default steps.\n- Bash is available by default in a per-sample temporary local directory; Docker sandbox and MCP servers are optional.\n- A single full run derives ``all``, ``en``, and ``zh`` reports without repeated inference.\n\n## Evaluation Notes\n\n- Uses the official Markdown table alignment and hybrid rule/LLM scoring semantics.\n- Requires ``judge.strategy='auto'`` or ``'llm'`` with at least one ``judge.models`` entry; rule-only scoring is unsupported.\n- See the [WideSearch usage guide](https://evalscope.readthedocs.io/en/latest/third_party/wide_search.html) for runtime\n examples and paper-style repeat settings.\n",
|
||
"prompt_template": "{question}",
|
||
"system_prompt": "",
|
||
"few_shot_prompt_template": "",
|
||
"aggregation": "mean",
|
||
"extra_params": {},
|
||
"sandbox_config": {},
|
||
"category": "agent",
|
||
"primary_metric": {
|
||
"name": "success_rate",
|
||
"aggregation": "pass_at_k",
|
||
"dimensions": {
|
||
"scope": "all"
|
||
}
|
||
},
|
||
"agent_config": {
|
||
"strategy": "function_calling",
|
||
"max_steps": 50
|
||
}
|
||
},
|
||
"statistics": {
|
||
"total_samples": 200,
|
||
"subset_stats": [
|
||
{
|
||
"name": "default",
|
||
"sample_count": 200,
|
||
"prompt_length_mean": 631.01,
|
||
"prompt_length_min": 182,
|
||
"prompt_length_max": 1807,
|
||
"prompt_length_std": 351.12,
|
||
"target_length_mean": 5759.28
|
||
}
|
||
],
|
||
"prompt_length": {
|
||
"mean": 631.01,
|
||
"min": 182,
|
||
"max": 1807,
|
||
"std": 351.12
|
||
},
|
||
"target_length_mean": 5759.28,
|
||
"computed_at": "2026-07-13T10:25:01.273130"
|
||
},
|
||
"sample_example": {
|
||
"data": {
|
||
"input": [
|
||
{
|
||
"id": "59f736f6",
|
||
"content": "My son is about to start his university applications in 2025 for postgraduates but he’s still uncertain about both his major and which universities to apply to. Could you help me find the top five universities in each of the five broad subjec ... [TRUNCATED 691 chars] ... names in English. \nUse only Arabic numerals in the ranking, for example: 1.\nDon't ask me any questions, just output the results according to the columns without omitting cells arbitrarily. The output format is \n```markdown\n{data_content}\n```."
|
||
}
|
||
],
|
||
"target": "Subject,University,Country,QS World University Rankings by Subject 2025,QS World University Rankings 2025,Times Higher Education World University Rankings 2025,Home Page,Application Deadline,Application Fee\nArts & Humanities,Harvard Universi ... [TRUNCATED 2837 chars] ... nuary 5,$90\nSocial Sciences & Management,Massachusetts Institute of Technology,United States,4,1,2,https://www.mit.edu/,January 6 ,$75\nSocial Sciences & Management,University of Cambridge,United Kingdom,5,5,5,https://www.cam.ac.uk/,Oct 15,£60",
|
||
"id": 0,
|
||
"group_id": 0,
|
||
"tools": [
|
||
{
|
||
"name": "bash",
|
||
"description": "Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.",
|
||
"parameters": {
|
||
"properties": {
|
||
"command": {
|
||
"type": "string",
|
||
"description": "The bash command to execute."
|
||
},
|
||
"timeout": {
|
||
"type": "number",
|
||
"description": "Maximum execution time in seconds (default: 60).",
|
||
"default": 60
|
||
}
|
||
},
|
||
"required": [
|
||
"command"
|
||
]
|
||
}
|
||
}
|
||
],
|
||
"metadata": {
|
||
"instance_id": "ws_en_001",
|
||
"language": "en",
|
||
"evaluation": {
|
||
"unique_columns": [
|
||
"subject",
|
||
"university"
|
||
],
|
||
"required": [
|
||
"subject",
|
||
"university",
|
||
"qsworlduniversityrankingsbysubject2025",
|
||
"qsworlduniversityrankings2025",
|
||
"timeshighereducationworlduniversityrankings2025",
|
||
"homepage",
|
||
"applicationdeadline",
|
||
"applicationfee"
|
||
],
|
||
"eval_pipeline": {
|
||
"applicationdeadline": {
|
||
"preprocess": [
|
||
"norm_str"
|
||
],
|
||
"metric": [
|
||
"llm_judge"
|
||
],
|
||
"criterion": "It is sufficient if the semantics are approximately the same as the reference answer or if they point to the same entity. There is no need for a word-for-word correspondence.\nThe month and day must be correct"
|
||
},
|
||
"applicationfee": {
|
||
"preprocess": [
|
||
"norm_str"
|
||
],
|
||
"metric": [
|
||
"llm_judge"
|
||
],
|
||
"criterion": "It is sufficient if the semantics are approximately the same as the reference answer or if they point to the same entity. There is no need for a word-for-word correspondence.\nIf there are multiple fees in the reference answer, all must be included."
|
||
},
|
||
"homepage": {
|
||
"preprocess": [
|
||
"norm_str"
|
||
],
|
||
"metric": [
|
||
"url_match"
|
||
]
|
||
},
|
||
"subject": {
|
||
"preprocess": [
|
||
"norm_str"
|
||
],
|
||
"metric": [
|
||
"exact_match"
|
||
]
|
||
},
|
||
"university": {
|
||
"preprocess": [
|
||
"norm_str"
|
||
],
|
||
"metric": [
|
||
"exact_match"
|
||
]
|
||
},
|
||
"qsworlduniversityrankingsbysubject2025": {
|
||
"preprocess": [
|
||
"norm_str"
|
||
],
|
||
"metric": [
|
||
"exact_match"
|
||
]
|
||
},
|
||
"qsworlduniversityrankings2025": {
|
||
"preprocess": [
|
||
"norm_str"
|
||
],
|
||
"metric": [
|
||
"exact_match"
|
||
]
|
||
},
|
||
"timeshighereducationworlduniversityrankings2025": {
|
||
"preprocess": [
|
||
"norm_str"
|
||
],
|
||
"metric": [
|
||
"exact_match"
|
||
]
|
||
}
|
||
}
|
||
}
|
||
}
|
||
},
|
||
"subset": "default",
|
||
"truncated": true
|
||
},
|
||
"readme": {
|
||
"en": "# WideSearch\n\n\n## Overview\n\nWideSearch evaluates search agents on broad web information-seeking tasks. Each task asks the agent to collect many\natomic facts and return one structured Markdown table. EvalScope uses the ModelScope\n`bytedance-community/WideSearch` dataset.\n\n## Task Description\n\n- **Task Type**: Multi-turn search agent\n- **Input**: Natural-language collection request with an explicit table schema\n- **Output**: Complete Markdown table\n- **Dataset**: 200 tasks in the ``full`` split; 100 English and 100 Chinese\n\n## Key Features\n\n- Official single-agent protocol: language-specific system prompt, ``function_calling``, and 50 default steps.\n- Bash is available by default in a per-sample temporary local directory; Docker sandbox and MCP servers are optional.\n- A single full run derives ``all``, ``en``, and ``zh`` reports without repeated inference.\n\n## Evaluation Notes\n\n- Uses the official Markdown table alignment and hybrid rule/LLM scoring semantics.\n- Requires ``judge.strategy='auto'`` or ``'llm'`` with at least one ``judge.models`` entry; rule-only scoring is unsupported.\n- See the [WideSearch usage guide](https://evalscope.readthedocs.io/en/latest/third_party/wide_search.html) for runtime\n examples and paper-style repeat settings.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `wide_search` |\n| **Dataset ID** | [bytedance-community/WideSearch](https://modelscope.cn/datasets/bytedance-community/WideSearch/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2508.07999) |\n| **Tags** | `Agent`, `MultiTurn`, `Retrieval` |\n| **Metrics** | `success_rate`, `row_precision`, `row_recall`, `row_f1`, `item_precision`, `item_recall`, `item_f1` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `full` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 200 |\n| Prompt Length (Mean) | 631.01 chars |\n| Prompt Length (Min/Max) | 182 / 1807 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"59f736f6\",\n \"content\": \"My son is about to start his university applications in 2025 for postgraduates but he’s still uncertain about both his major and which universities to apply to. Could you help me find the top five universities in each of the five broad subjec ... [TRUNCATED 691 chars] ... names in English. \\nUse only Arabic numerals in the ranking, for example: 1.\\nDon't ask me any questions, just output the results according to the columns without omitting cells arbitrarily. The output format is \\n```markdown\\n{data_content}\\n```.\"\n }\n ],\n \"target\": \"Subject,University,Country,QS World University Rankings by Subject 2025,QS World University Rankings 2025,Times Higher Education World University Rankings 2025,Home Page,Application Deadline,Application Fee\\nArts & Humanities,Harvard Universi ... [TRUNCATED 2837 chars] ... nuary 5,$90\\nSocial Sciences & Management,Massachusetts Institute of Technology,United States,4,1,2,https://www.mit.edu/,January 6 ,$75\\nSocial Sciences & Management,University of Cambridge,United Kingdom,5,5,5,https://www.cam.ac.uk/,Oct 15,£60\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"instance_id\": \"ws_en_001\",\n \"language\": \"en\",\n \"evaluation\": {\n \"unique_columns\": [\n \"subject\",\n \"university\"\n ],\n \"required\": [\n \"subject\",\n \"university\",\n \"qsworlduniversityrankingsbysubject2025\",\n \"qsworlduniversityrankings2025\",\n \"timeshighereducationworlduniversityrankings2025\",\n \"homepage\",\n \"applicationdeadline\",\n \"applicationfee\"\n ],\n \"eval_pipeline\": {\n \"applicationdeadline\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"llm_judge\"\n ],\n \"criterion\": \"It is sufficient if the semantics are approximately the same as the reference answer or if they point to the same entity. There is no need for a word-for-word correspondence.\\nThe month and day must be correct\"\n },\n \"applicationfee\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"llm_judge\"\n ],\n \"criterion\": \"It is sufficient if the semantics are approximately the same as the reference answer or if they point to the same entity. There is no need for a word-for-word correspondence.\\nIf there are multiple fees in the reference answer, all must be included.\"\n },\n \"homepage\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"url_match\"\n ]\n },\n \"subject\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"university\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"qsworlduniversityrankingsbysubject2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"qsworlduniversityrankings2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"timeshighereducationworlduniversityrankings2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n }\n }\n }\n }\n}\n```\n\n*Note: Some content was truncated for display.*\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets wide_search \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":50}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['wide_search'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=50,\n ),\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n",
|
||
"zh": "# WideSearch\n\n\n## 概述\n\nWideSearch 用于评估搜索智能体在广泛网络信息检索任务上的表现。每个任务要求智能体收集多个原子事实，并返回一个结构化的 Markdown 表格。EvalScope 使用 ModelScope 上的 `bytedance-community/WideSearch` 数据集。\n\n## 任务描述\n\n- **任务类型**：多轮搜索智能体\n- **输入**：包含显式表格结构的自然语言信息收集请求\n- **输出**：完整的 Markdown 表格\n- **数据集**：``full`` 划分中共有 200 个任务；其中 100 个为英文，100 个为中文\n\n## 核心特性\n\n- 官方单智能体协议：语言特定的系统提示词、``function_calling`` 机制，以及默认 50 步的最大执行步数。\n- 默认为每个样本提供一个临时本地目录并启用 Bash；Docker 沙箱和 MCP 服务器为可选项。\n- 单次完整运行即可生成 ``all``、``en`` 和 ``zh`` 三份报告，无需重复推理。\n\n## 评估说明\n\n- 使用官方的 Markdown 表格对齐方式及混合规则/大语言模型（LLM）评分语义。\n- 要求设置 ``judge.strategy='auto'`` 或 ``'llm'``，且至少指定一个 ``judge.models`` 条目；不支持仅使用规则评分。\n- 运行时示例及论文风格的重复实验设置，请参阅 [WideSearch 使用指南](https://evalscope.readthedocs.io/zh-cn/latest/third_party/wide_search.html)。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `wide_search` |\n| **数据集ID** | [bytedance-community/WideSearch](https://modelscope.cn/datasets/bytedance-community/WideSearch/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2508.07999) |\n| **标签** | `Agent`, `MultiTurn`, `Retrieval` |\n| **指标** | `success_rate`, `row_precision`, `row_recall`, `row_f1`, `item_precision`, `item_recall`, `item_f1` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `full` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 200 |\n| 提示词长度（平均） | 631.01 字符 |\n| 提示词长度（最小/最大） | 182 / 1807 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"59f736f6\",\n \"content\": \"My son is about to start his university applications in 2025 for postgraduates but he’s still uncertain about both his major and which universities to apply to. Could you help me find the top five universities in each of the five broad subjec ... [TRUNCATED 691 chars] ... names in English. \\nUse only Arabic numerals in the ranking, for example: 1.\\nDon't ask me any questions, just output the results according to the columns without omitting cells arbitrarily. The output format is \\n```markdown\\n{data_content}\\n```.\"\n }\n ],\n \"target\": \"Subject,University,Country,QS World University Rankings by Subject 2025,QS World University Rankings 2025,Times Higher Education World University Rankings 2025,Home Page,Application Deadline,Application Fee\\nArts & Humanities,Harvard Universi ... [TRUNCATED 2837 chars] ... nuary 5,$90\\nSocial Sciences & Management,Massachusetts Institute of Technology,United States,4,1,2,https://www.mit.edu/,January 6 ,$75\\nSocial Sciences & Management,University of Cambridge,United Kingdom,5,5,5,https://www.cam.ac.uk/,Oct 15,£60\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"instance_id\": \"ws_en_001\",\n \"language\": \"en\",\n \"evaluation\": {\n \"unique_columns\": [\n \"subject\",\n \"university\"\n ],\n \"required\": [\n \"subject\",\n \"university\",\n \"qsworlduniversityrankingsbysubject2025\",\n \"qsworlduniversityrankings2025\",\n \"timeshighereducationworlduniversityrankings2025\",\n \"homepage\",\n \"applicationdeadline\",\n \"applicationfee\"\n ],\n \"eval_pipeline\": {\n \"applicationdeadline\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"llm_judge\"\n ],\n \"criterion\": \"It is sufficient if the semantics are approximately the same as the reference answer or if they point to the same entity. There is no need for a word-for-word correspondence.\\nThe month and day must be correct\"\n },\n \"applicationfee\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"llm_judge\"\n ],\n \"criterion\": \"It is sufficient if the semantics are approximately the same as the reference answer or if they point to the same entity. There is no need for a word-for-word correspondence.\\nIf there are multiple fees in the reference answer, all must be included.\"\n },\n \"homepage\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"url_match\"\n ]\n },\n \"subject\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"university\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"qsworlduniversityrankingsbysubject2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"qsworlduniversityrankings2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"timeshighereducationworlduniversityrankings2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n }\n }\n }\n }\n}\n```\n\n*注：部分内容因展示需要已被截断。*\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets wide_search \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":50}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['wide_search'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=50,\n ),\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```",
|
||
"content_hash": "0d89fb2f4e0b14234fa84358e3abc97a",
|
||
"needs_translation": false
|
||
},
|
||
"updated_at": "2026-08-20T10:47:12.857971",
|
||
"translation_updated_at": "2026-08-20T10:47:18"
|
||
}
|