13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
197 lines
20 KiB
JSON
197 lines
20 KiB
JSON
{
|
||
"meta": {
|
||
"pretty_name": "OmniDocBench",
|
||
"dataset_id": "evalscope/OmniDocBench_tsv",
|
||
"paper_url": null,
|
||
"tags": [
|
||
"MultiModal",
|
||
"Knowledge",
|
||
"QA"
|
||
],
|
||
"metrics": [
|
||
{
|
||
"text_block": {
|
||
"metric": [
|
||
"Edit_dist",
|
||
"BLEU",
|
||
"METEOR"
|
||
]
|
||
}
|
||
},
|
||
{
|
||
"display_formula": {
|
||
"metric": [
|
||
"Edit_dist"
|
||
]
|
||
}
|
||
},
|
||
{
|
||
"table": {
|
||
"metric": [
|
||
"TEDS",
|
||
"Edit_dist"
|
||
]
|
||
}
|
||
},
|
||
{
|
||
"reading_order": {
|
||
"metric": [
|
||
"Edit_dist"
|
||
]
|
||
}
|
||
},
|
||
{
|
||
"normalized_score": {
|
||
"metric": []
|
||
}
|
||
}
|
||
],
|
||
"primary_metric": {
|
||
"name": "normalized_score",
|
||
"aggregation": "macro_mean",
|
||
"dimensions": {}
|
||
},
|
||
"few_shot_num": 0,
|
||
"eval_split": "train",
|
||
"train_split": "",
|
||
"subset_list": [
|
||
"default"
|
||
],
|
||
"description": "\n## Overview\n\nThis adapter preserves EvalScope's original 981-page OmniDocBench TSV integration for compatibility with existing evaluations.\n\n## Task Description\n\n- **Task Type**: Document Parsing and Understanding\n- **Input**: PDF page image\n- **Output**: Parsed document structure in Markdown format\n- **Domain**: Document understanding, OCR, layout analysis\n\n## Key Features\n\n- Uses the legacy `evalscope/OmniDocBench_tsv` dataset with 981 PDF pages\n- Covers text blocks, formulas, tables, and reading order\n- Keeps the existing local Python scoring implementation and metric names unchanged\n- Remains available for reproducing existing EvalScope results\n\n## Evaluation Notes\n\n- This legacy TSV dataset is not labeled as a specific upstream OmniDocBench release.\n- For new evaluations, use the recommended `omni_doc_bench_v1_6` benchmark.\n- Implements the existing `end2end` and `quick_match` scoring paths.\n- Metrics: Edit_dist, BLEU, METEOR (text), TEDS (tables)\n- Install the `evalscope[omnidoc_bench]` extra for legacy scoring dependencies.\n- Output format: Markdown with LaTeX formulas and HTML tables\n- Scores from this legacy integration are not directly comparable with v1.6 scores.\n",
|
||
"prompt_template": " You are an AI assistant specialized in converting PDF images to Markdown format. Please follow these instructions for the conversion:\n\n 1. Text Processing:\n - Accurately recognize all text content in the PDF image without guessing or inferring.\n - Convert the recognized text into Markdown format.\n - Maintain the original document structure, including headings, paragraphs, lists, etc.\n\n 2. Mathematical Formula Processing:\n - Convert all mathematical formulas to LaTeX format.\n - Enclose inline formulas with \\( \\). For example: This is an inline formula \\( E = mc^2 \\)\n - Enclose block formulas with \\\\[ \\\\]. For example: \\[ \\frac{-b \\pm \\sqrt{b^2 - 4ac}}{2a} \\]\n\n 3. Table Processing:\n - Convert tables to HTML format.\n - Wrap the entire table with <table> and </table>.\n\n 4. Figure Handling:\n - Ignore figures content in the PDF image. Do not attempt to describe or convert images.\n\n 5. Output Format:\n - Ensure the output Markdown document has a clear structure with appropriate line breaks between elements.\n - For complex layouts, try to maintain the original document's structure and format as closely as possible.\n\n Please strictly follow these guidelines to ensure accuracy and consistency in the conversion. Your task is to accurately convert the content of the PDF image into Markdown format without adding any extra explanations or comments.\n",
|
||
"system_prompt": "",
|
||
"few_shot_prompt_template": "",
|
||
"aggregation": "mean",
|
||
"extra_params": {
|
||
"match_method": {
|
||
"type": "str",
|
||
"description": "Scoring match method used for evaluation.",
|
||
"value": "quick_match",
|
||
"choices": [
|
||
"quick_match",
|
||
"simple_match",
|
||
"no_split"
|
||
]
|
||
}
|
||
},
|
||
"sandbox_config": {},
|
||
"category": "vlm"
|
||
},
|
||
"statistics": {
|
||
"total_samples": 981,
|
||
"subset_stats": [
|
||
{
|
||
"name": "default",
|
||
"sample_count": 981,
|
||
"prompt_length_mean": 1408,
|
||
"prompt_length_min": 1408,
|
||
"prompt_length_max": 1408,
|
||
"prompt_length_std": null,
|
||
"target_length_mean": 40340.42,
|
||
"multimodal": {
|
||
"has_images": true,
|
||
"has_audio": false,
|
||
"has_video": false,
|
||
"image": {
|
||
"count_total": 981,
|
||
"count_per_sample": {
|
||
"min": 1,
|
||
"max": 1,
|
||
"mean": 1
|
||
},
|
||
"resolutions": [
|
||
"10142x14342",
|
||
"1024x1457",
|
||
"1102x1631",
|
||
"1102x1646",
|
||
"11112x7925",
|
||
"1142x1654",
|
||
"1156x1637",
|
||
"1159x1639",
|
||
"1181x1709",
|
||
"1189x1701"
|
||
],
|
||
"resolution_range": {
|
||
"min": "516x729",
|
||
"max": "10142x14342"
|
||
},
|
||
"formats": [
|
||
"jpeg"
|
||
]
|
||
}
|
||
}
|
||
}
|
||
],
|
||
"prompt_length": {
|
||
"mean": 1408,
|
||
"min": 1408,
|
||
"max": 1408,
|
||
"std": null
|
||
},
|
||
"target_length_mean": 40340.42,
|
||
"computed_at": "2026-08-04T14:31:09.382977",
|
||
"multimodal": {
|
||
"has_images": true,
|
||
"has_audio": false,
|
||
"has_video": false,
|
||
"image": {
|
||
"count_total": 981,
|
||
"count_per_sample": {
|
||
"min": 1,
|
||
"max": 1,
|
||
"mean": 1
|
||
},
|
||
"resolutions": [
|
||
"10142x14342",
|
||
"1024x1457",
|
||
"1102x1631",
|
||
"1102x1646",
|
||
"11112x7925",
|
||
"1142x1654",
|
||
"1156x1637",
|
||
"1159x1639",
|
||
"1181x1709",
|
||
"1189x1701"
|
||
],
|
||
"resolution_range": {
|
||
"min": "516x729",
|
||
"max": "10142x14342"
|
||
},
|
||
"formats": [
|
||
"jpeg"
|
||
]
|
||
}
|
||
}
|
||
},
|
||
"sample_example": {
|
||
"data": {
|
||
"input": [
|
||
{
|
||
"id": "7c6fda98",
|
||
"content": [
|
||
{
|
||
"image": "[BASE64_IMAGE: png, ~321.8KB]"
|
||
},
|
||
{
|
||
"text": " You are an AI assistant specialized in converting PDF images to Markdown format. Please follow these instructions for the conversion:\n\n 1. Text Processing:\n - Accurately recognize all text content in the PDF image without guessing or i ... [TRUNCATED 924 chars] ... sible.\n\n Please strictly follow these guidelines to ensure accuracy and consistency in the conversion. Your task is to accurately convert the content of the PDF image into Markdown format without adding any extra explanations or comments.\n"
|
||
}
|
||
]
|
||
}
|
||
],
|
||
"target": "{\"layout_dets\": [{\"category_type\": \"title\", \"poly\": [102.5999912116609, 120.87255879760278, 719.3118659856144, 120.87255879760278, 719.3118659856144, 194.14083813380114, 102.5999912116609, 194.14083813380114], \"ignore\": false, \"order\": 1, \"an ... [TRUNCATED 9876 chars] ... nguage\": \"simplified_chinese\", \"layout\": \"1andmore_column\", \"special_issue\": [\"watermark\"]}, \"page_no\": 11, \"height\": 1500, \"width\": 2667, \"image_path\": \"eastmoney_59cde7e939acc3124df9d3f2c85b5a0ec41b9da1157d5be38e098672022b47cb.pdf_11.jpg\"}}",
|
||
"id": 0,
|
||
"group_id": 0
|
||
},
|
||
"subset": "default",
|
||
"truncated": true
|
||
},
|
||
"readme": {
|
||
"en": "# OmniDocBench\n\n\n## Overview\n\nThis adapter preserves EvalScope's original 981-page OmniDocBench TSV integration for compatibility with existing evaluations.\n\n## Task Description\n\n- **Task Type**: Document Parsing and Understanding\n- **Input**: PDF page image\n- **Output**: Parsed document structure in Markdown format\n- **Domain**: Document understanding, OCR, layout analysis\n\n## Key Features\n\n- Uses the legacy `evalscope/OmniDocBench_tsv` dataset with 981 PDF pages\n- Covers text blocks, formulas, tables, and reading order\n- Keeps the existing local Python scoring implementation and metric names unchanged\n- Remains available for reproducing existing EvalScope results\n\n## Evaluation Notes\n\n- This legacy TSV dataset is not labeled as a specific upstream OmniDocBench release.\n- For new evaluations, use the recommended `omni_doc_bench_v1_6` benchmark.\n- Implements the existing `end2end` and `quick_match` scoring paths.\n- Metrics: Edit_dist, BLEU, METEOR (text), TEDS (tables)\n- Install the `evalscope[omnidoc_bench]` extra for legacy scoring dependencies.\n- Output format: Markdown with LaTeX formulas and HTML tables\n- Scores from this legacy integration are not directly comparable with v1.6 scores.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `omni_doc_bench` |\n| **Dataset ID** | [evalscope/OmniDocBench_tsv](https://modelscope.cn/datasets/evalscope/OmniDocBench_tsv/summary) |\n| **Paper** | N/A |\n| **Tags** | `Knowledge`, `MultiModal`, `QA` |\n| **Metrics** | `text_block`, `display_formula`, `table`, `reading_order`, `normalized_score` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 981 |\n| Prompt Length (Mean) | 1408 chars |\n| Prompt Length (Min/Max) | 1408 / 1408 chars |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 981 |\n| Images per Sample | min: 1, max: 1, mean: 1 |\n| Resolution Range | 516x729 - 10142x14342 |\n| Formats | jpeg |\n\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"7c6fda98\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: png, ~321.8KB]\"\n },\n {\n \"text\": \" You are an AI assistant specialized in converting PDF images to Markdown format. Please follow these instructions for the conversion:\\n\\n 1. Text Processing:\\n - Accurately recognize all text content in the PDF image without guessing or i ... [TRUNCATED 924 chars] ... sible.\\n\\n Please strictly follow these guidelines to ensure accuracy and consistency in the conversion. Your task is to accurately convert the content of the PDF image into Markdown format without adding any extra explanations or comments.\\n\"\n }\n ]\n }\n ],\n \"target\": \"{\\\"layout_dets\\\": [{\\\"category_type\\\": \\\"title\\\", \\\"poly\\\": [102.5999912116609, 120.87255879760278, 719.3118659856144, 120.87255879760278, 719.3118659856144, 194.14083813380114, 102.5999912116609, 194.14083813380114], \\\"ignore\\\": false, \\\"order\\\": 1, \\\"an ... [TRUNCATED 9876 chars] ... nguage\\\": \\\"simplified_chinese\\\", \\\"layout\\\": \\\"1andmore_column\\\", \\\"special_issue\\\": [\\\"watermark\\\"]}, \\\"page_no\\\": 11, \\\"height\\\": 1500, \\\"width\\\": 2667, \\\"image_path\\\": \\\"eastmoney_59cde7e939acc3124df9d3f2c85b5a0ec41b9da1157d5be38e098672022b47cb.pdf_11.jpg\\\"}}\",\n \"id\": 0,\n \"group_id\": 0\n}\n```\n\n*Note: Some content was truncated for display.*\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n You are an AI assistant specialized in converting PDF images to Markdown format. Please follow these instructions for the conversion:\n\n 1. Text Processing:\n - Accurately recognize all text content in the PDF image without guessing or inferring.\n - Convert the recognized text into Markdown format.\n - Maintain the original document structure, including headings, paragraphs, lists, etc.\n\n 2. Mathematical Formula Processing:\n - Convert all mathematical formulas to LaTeX format.\n - Enclose inline formulas with \\( \\). For example: This is an inline formula \\( E = mc^2 \\)\n - Enclose block formulas with \\\\[ \\\\]. For example: \\[ \\frac{-b \\pm \\sqrt{b^2 - 4ac}}{2a} \\]\n\n 3. Table Processing:\n - Convert tables to HTML format.\n - Wrap the entire table with <table> and </table>.\n\n 4. Figure Handling:\n - Ignore figures content in the PDF image. Do not attempt to describe or convert images.\n\n 5. Output Format:\n - Ensure the output Markdown document has a clear structure with appropriate line breaks between elements.\n - For complex layouts, try to maintain the original document's structure and format as closely as possible.\n\n Please strictly follow these guidelines to ensure accuracy and consistency in the conversion. Your task is to accurately convert the content of the PDF image into Markdown format without adding any extra explanations or comments.\n\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `match_method` | `str` | `quick_match` | Scoring match method used for evaluation. Choices: ['quick_match', 'simple_match', 'no_split'] |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets omni_doc_bench \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['omni_doc_bench'],\n dataset_args={\n 'omni_doc_bench': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n",
|
||
"zh": "# OmniDocBench\n\n\n## 概述\n\n此适配器保留了 EvalScope 原始的 981 页 OmniDocBench TSV 集成，以确保与现有评估结果兼容。\n\n## 任务描述\n\n- **任务类型**：文档解析与理解\n- **输入**：PDF 页面图像\n- **输出**：以 Markdown 格式表示的解析后文档结构\n- **领域**：文档理解、OCR、版面分析\n\n## 主要特性\n\n- 使用旧版 `evalscope/OmniDocBench_tsv` 数据集，包含 981 个 PDF 页面\n- 覆盖文本块、公式、表格和阅读顺序\n- 保留现有的本地 Python 评分实现和指标名称不变\n- 可用于复现现有的 EvalScope 评估结果\n\n## 评估说明\n\n- 此旧版 TSV 数据集未标记为特定的上游 OmniDocBench 发布版本。\n- 对于新评估，请使用推荐的 `omni_doc_bench_v1_6` 基准测试。\n- 实现了现有的 `end2end` 和 `quick_match` 评分路径。\n- 指标：Edit_dist、BLEU、METEOR（文本）、TEDS（表格）\n- 安装 `evalscope[omnidoc_bench]` 额外依赖项以支持旧版评分功能。\n- 输出格式：包含 LaTeX 公式和 HTML 表格的 Markdown\n- 此旧版集成产生的分数无法直接与 v1.6 版本的分数进行比较。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `omni_doc_bench` |\n| **数据集ID** | [evalscope/OmniDocBench_tsv](https://modelscope.cn/datasets/evalscope/OmniDocBench_tsv/summary) |\n| **论文** | N/A |\n| **标签** | `Knowledge`, `MultiModal`, `QA` |\n| **指标** | `text_block`, `display_formula`, `table`, `reading_order`, `normalized_score` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `train` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 981 |\n| 提示词长度（平均） | 1408 字符 |\n| 提示词长度（最小/最大） | 1408 / 1408 字符 |\n\n**图像统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 981 |\n| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 |\n| 分辨率范围 | 516x729 - 10142x14342 |\n| 格式 | jpeg |\n\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"7c6fda98\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: png, ~321.8KB]\"\n },\n {\n \"text\": \" You are an AI assistant specialized in converting PDF images to Markdown format. Please follow these instructions for the conversion:\\n\\n 1. Text Processing:\\n - Accurately recognize all text content in the PDF image without guessing or i ... [TRUNCATED 924 chars] ... sible.\\n\\n Please strictly follow these guidelines to ensure accuracy and consistency in the conversion. Your task is to accurately convert the content of the PDF image into Markdown format without adding any extra explanations or comments.\\n\"\n }\n ]\n }\n ],\n \"target\": \"{\\\"layout_dets\\\": [{\\\"category_type\\\": \\\"title\\\", \\\"poly\\\": [102.5999912116609, 120.87255879760278, 719.3118659856144, 120.87255879760278, 719.3118659856144, 194.14083813380114, 102.5999912116609, 194.14083813380114], \\\"ignore\\\": false, \\\"order\\\": 1, \\\"an ... [TRUNCATED 9876 chars] ... nguage\\\": \\\"simplified_chinese\\\", \\\"layout\\\": \\\"1andmore_column\\\", \\\"special_issue\\\": [\\\"watermark\\\"]}, \\\"page_no\\\": 11, \\\"height\\\": 1500, \\\"width\\\": 2667, \\\"image_path\\\": \\\"eastmoney_59cde7e939acc3124df9d3f2c85b5a0ec41b9da1157d5be38e098672022b47cb.pdf_11.jpg\\\"}}\",\n \"id\": 0,\n \"group_id\": 0\n}\n```\n\n*注：部分内容因显示需要已被截断。*\n\n## 提示模板\n\n**提示模板：**\n```text\n You are an AI assistant specialized in converting PDF images to Markdown format. Please follow these instructions for the conversion:\n\n 1. Text Processing:\n - Accurately recognize all text content in the PDF image without guessing or inferring.\n - Convert the recognized text into Markdown format.\n - Maintain the original document structure, including headings, paragraphs, lists, etc.\n\n 2. Mathematical Formula Processing:\n - Convert all mathematical formulas to LaTeX format.\n - Enclose inline formulas with \\( \\). For example: This is an inline formula \\( E = mc^2 \\)\n - Enclose block formulas with \\\\[ \\\\]. For example: \\[ \\frac{-b \\pm \\sqrt{b^2 - 4ac}}{2a} \\]\n\n 3. Table Processing:\n - Convert tables to HTML format.\n - Wrap the entire table with <table> and </table>.\n\n 4. Figure Handling:\n - Ignore figures content in the PDF image. Do not attempt to describe or convert images.\n\n 5. Output Format:\n - Ensure the output Markdown document has a clear structure with appropriate line breaks between elements.\n - For complex layouts, try to maintain the original document's structure and format as closely as possible.\n\n Please strictly follow these guidelines to ensure accuracy and consistency in the conversion. Your task is to accurately convert the content of the PDF image into Markdown format without adding any extra explanations or comments.\n\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `match_method` | `str` | `quick_match` | 评估时使用的评分匹配方法。可选值：['quick_match', 'simple_match', 'no_split'] |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets omni_doc_bench \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['omni_doc_bench'],\n dataset_args={\n 'omni_doc_bench': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```",
|
||
"content_hash": "46a9ae26d528184b9a11e74a298bc26a",
|
||
"needs_translation": false
|
||
},
|
||
"updated_at": "2026-08-12T10:59:03.167232",
|
||
"translation_updated_at": "2026-08-04T14:31:11"
|
||
}
|