13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
456 lines
44 KiB
JSON
456 lines
44 KiB
JSON
{
|
||
"meta": {
|
||
"pretty_name": "ACEBench",
|
||
"dataset_id": "evalscope/acebench",
|
||
"paper_url": null,
|
||
"tags": [
|
||
"FunctionCalling",
|
||
"Agent",
|
||
"MultiTurn"
|
||
],
|
||
"metrics": [
|
||
"accuracy",
|
||
"process_acc"
|
||
],
|
||
"primary_metric": {
|
||
"name": "accuracy",
|
||
"aggregation": null,
|
||
"dimensions": {}
|
||
},
|
||
"few_shot_num": 0,
|
||
"eval_split": "normal",
|
||
"train_split": "",
|
||
"subset_list": [
|
||
"normal_single_turn_single_function",
|
||
"normal_single_turn_parallel_function",
|
||
"normal_multi_turn_user_adjust",
|
||
"normal_multi_turn_user_switch",
|
||
"normal_similar_api",
|
||
"normal_preference",
|
||
"normal_atom_bool",
|
||
"normal_atom_enum",
|
||
"normal_atom_number",
|
||
"normal_atom_list",
|
||
"normal_atom_object_deep",
|
||
"normal_atom_object_short",
|
||
"special_incomplete",
|
||
"special_error_param",
|
||
"special_irrelevant",
|
||
"agent_multi_step",
|
||
"agent_multi_turn"
|
||
],
|
||
"description": "\n## Overview\n\nACEBench evaluates whether large language models can use tools in realistic settings: picking the\nright API, filling its arguments, pushing back on requests that cannot be satisfied, and driving\nmulti-step agent tasks against a simulated environment. Data is split into three families -\n`normal` (ordinary tool use), `special` (incomplete, incorrect or out-of-scope requests) and\n`agent` (multi-step and multi-turn interaction) - reported over 17 fine-grained categories.\n\n## Task Description\n\n- **Task Type**: Function calling and agentic tool use\n- **Input**: Conversation history, API specifications, and optional time or character-profile context\n- **Output**: A `[ApiName(key='value')]` call list, a diagnostic sentence, or a full agent trajectory\n- **Domain**: 8 domains and 68 sub-domains including technology, finance, health and society\n\n## Key Features\n\n- 1023 English and 1017 Chinese samples, selectable through `extra_params.language`.\n- Uses the official ACEBench prompts and the official `[ApiName(...)]` output contract, so an\n output that cannot be decoded scores zero instead of being rescued by lenient parsing.\n- `normal_multi_turn_*` categories are scored per dialogue: every step must be correct for the\n dialogue to count, matching the official turn-level aggregation.\n- `agent` categories run a real rollout against ACEBench's simulated phone, food-delivery and\n travel APIs, and are graded on the resulting environment state.\n\n## Evaluation Notes\n\n- `accuracy` is the primary metric. For `normal` and `special` it is answer accuracy; for `agent` it is\n end-state accuracy. `process_acc` additionally reports milestone progress for `agent` samples and\n per-step progress for `normal_multi_turn_*` samples.\n- The report adds the official groupings (ATOM, SINGLE_TURN, MULTI_TURN, NORMAL, SPECIAL, AGENT)\n and an OVERALL score weighted `normal` 0.578 / `special` 0.2676 / `agent` 0.1545. Weights are\n renormalized over the groups actually evaluated, so a partial run stays interpretable.\n- `agent_multi_turn` additionally needs a user simulator; set `extra_params.user_model` to the model\n that should play the user (the official runner uses `gpt-4o`). Without it those rollouts fail and\n score zero, so configure it before reading an OVERALL number.\n",
|
||
"prompt_template": "",
|
||
"system_prompt": "",
|
||
"few_shot_prompt_template": "",
|
||
"aggregation": "mean",
|
||
"extra_params": {
|
||
"language": {
|
||
"type": "str",
|
||
"description": "Dataset language to evaluate, either `en` or `zh`.",
|
||
"value": "en"
|
||
},
|
||
"user_model": {
|
||
"type": "str",
|
||
"description": "Model that plays the user in `agent_multi_turn` rollouts, e.g. `gpt-4o`. Those rollouts fail and score zero when unset.",
|
||
"value": ""
|
||
},
|
||
"user_model_api_url": {
|
||
"type": "str",
|
||
"description": "Base URL for `user_model`. Defaults to `MODELSCOPE_API_BASE`.",
|
||
"value": ""
|
||
},
|
||
"user_model_api_key": {
|
||
"type": "str",
|
||
"description": "API key for `user_model`. Defaults to `MODELSCOPE_SDK_TOKEN`.",
|
||
"value": ""
|
||
},
|
||
"max_dialog_turns": {
|
||
"type": "int",
|
||
"description": "Maximum number of agent rollout steps.",
|
||
"value": 40
|
||
}
|
||
},
|
||
"sandbox_config": {},
|
||
"category": "agent"
|
||
},
|
||
"statistics": {
|
||
"total_samples": 1023,
|
||
"subset_stats": [
|
||
{
|
||
"name": "normal_single_turn_single_function",
|
||
"sample_count": 100,
|
||
"prompt_length_mean": 5165.79,
|
||
"prompt_length_min": 2461,
|
||
"prompt_length_max": 9553,
|
||
"prompt_length_std": 2176.06,
|
||
"target_length_mean": 283.64
|
||
},
|
||
{
|
||
"name": "normal_single_turn_parallel_function",
|
||
"sample_count": 100,
|
||
"prompt_length_mean": 5036.21,
|
||
"prompt_length_min": 2295,
|
||
"prompt_length_max": 9644,
|
||
"prompt_length_std": 2028.11,
|
||
"target_length_mean": 562.33
|
||
},
|
||
{
|
||
"name": "normal_multi_turn_user_adjust",
|
||
"sample_count": 123,
|
||
"prompt_length_mean": 4658.51,
|
||
"prompt_length_min": 3172,
|
||
"prompt_length_max": 6976,
|
||
"prompt_length_std": 715.8,
|
||
"target_length_mean": 231.79
|
||
},
|
||
{
|
||
"name": "normal_multi_turn_user_switch",
|
||
"sample_count": 100,
|
||
"prompt_length_mean": 7546.46,
|
||
"prompt_length_min": 3467,
|
||
"prompt_length_max": 11835,
|
||
"prompt_length_std": 2051.15,
|
||
"target_length_mean": 246.65
|
||
},
|
||
{
|
||
"name": "normal_similar_api",
|
||
"sample_count": 50,
|
||
"prompt_length_mean": 3511.84,
|
||
"prompt_length_min": 2484,
|
||
"prompt_length_max": 6209,
|
||
"prompt_length_std": 729.41,
|
||
"target_length_mean": 186.06
|
||
},
|
||
{
|
||
"name": "normal_preference",
|
||
"sample_count": 50,
|
||
"prompt_length_mean": 8637.66,
|
||
"prompt_length_min": 7107,
|
||
"prompt_length_max": 10381,
|
||
"prompt_length_std": 846.46,
|
||
"target_length_mean": 188.26
|
||
},
|
||
{
|
||
"name": "normal_atom_bool",
|
||
"sample_count": 50,
|
||
"prompt_length_mean": 7377.62,
|
||
"prompt_length_min": 4762,
|
||
"prompt_length_max": 9727,
|
||
"prompt_length_std": 1421.02,
|
||
"target_length_mean": 144.4
|
||
},
|
||
{
|
||
"name": "normal_atom_enum",
|
||
"sample_count": 50,
|
||
"prompt_length_mean": 7676.94,
|
||
"prompt_length_min": 4927,
|
||
"prompt_length_max": 11337,
|
||
"prompt_length_std": 1672.15,
|
||
"target_length_mean": 139.34
|
||
},
|
||
{
|
||
"name": "normal_atom_number",
|
||
"sample_count": 50,
|
||
"prompt_length_mean": 7481.46,
|
||
"prompt_length_min": 4851,
|
||
"prompt_length_max": 10278,
|
||
"prompt_length_std": 1757.87,
|
||
"target_length_mean": 123.2
|
||
},
|
||
{
|
||
"name": "normal_atom_list",
|
||
"sample_count": 50,
|
||
"prompt_length_mean": 7524.06,
|
||
"prompt_length_min": 4910,
|
||
"prompt_length_max": 10514,
|
||
"prompt_length_std": 1669.48,
|
||
"target_length_mean": 148.92
|
||
},
|
||
{
|
||
"name": "normal_atom_object_deep",
|
||
"sample_count": 50,
|
||
"prompt_length_mean": 6102.02,
|
||
"prompt_length_min": 2873,
|
||
"prompt_length_max": 9755,
|
||
"prompt_length_std": 2019.7,
|
||
"target_length_mean": 358.5
|
||
},
|
||
{
|
||
"name": "normal_atom_object_short",
|
||
"sample_count": 50,
|
||
"prompt_length_mean": 5139.5,
|
||
"prompt_length_min": 2343,
|
||
"prompt_length_max": 8921,
|
||
"prompt_length_std": 1935.93,
|
||
"target_length_mean": 255.78
|
||
},
|
||
{
|
||
"name": "special_incomplete",
|
||
"sample_count": 50,
|
||
"prompt_length_mean": 6177.34,
|
||
"prompt_length_min": 3473,
|
||
"prompt_length_max": 10806,
|
||
"prompt_length_std": 2108.09,
|
||
"target_length_mean": 106.12
|
||
},
|
||
{
|
||
"name": "special_error_param",
|
||
"sample_count": 50,
|
||
"prompt_length_mean": 4499.78,
|
||
"prompt_length_min": 3121,
|
||
"prompt_length_max": 6090,
|
||
"prompt_length_std": 1006.27,
|
||
"target_length_mean": 65.84
|
||
},
|
||
{
|
||
"name": "special_irrelevant",
|
||
"sample_count": 50,
|
||
"prompt_length_mean": 6011.94,
|
||
"prompt_length_min": 3778,
|
||
"prompt_length_max": 8492,
|
||
"prompt_length_std": 1395.87,
|
||
"target_length_mean": 106
|
||
},
|
||
{
|
||
"name": "agent_multi_step",
|
||
"sample_count": 20,
|
||
"prompt_length_mean": 6407.9,
|
||
"prompt_length_min": 6343,
|
||
"prompt_length_max": 6472,
|
||
"prompt_length_std": 40.01,
|
||
"target_length_mean": 2138.4
|
||
},
|
||
{
|
||
"name": "agent_multi_turn",
|
||
"sample_count": 30,
|
||
"prompt_length_mean": 6290.97,
|
||
"prompt_length_min": 5505,
|
||
"prompt_length_max": 6630,
|
||
"prompt_length_std": 388.89,
|
||
"target_length_mean": 1942.2
|
||
}
|
||
],
|
||
"prompt_length": {
|
||
"mean": 6032.98,
|
||
"min": 2295,
|
||
"max": 11835,
|
||
"std": 2119.04
|
||
},
|
||
"target_length_mean": 322.51,
|
||
"computed_at": "2026-08-05T14:52:55.848799"
|
||
},
|
||
"sample_example": {
|
||
"data": {
|
||
"input": [
|
||
{
|
||
"id": "9198db95",
|
||
"content": "You are an AI assistant with the role name \"assistant.\" Based on the provided API specifications and conversation history from steps 1 to t, generate the API requests that the assistant should call in step t+1. The API requests should be outp ... [TRUNCATED 3788 chars] ... '}, 'effects': {'description': 'List of audio effects to apply.', 'type': 'array', 'items': {'type': 'string', 'enum': ['reverb', 'echo', 'distortion']}}}, 'required': ['frequency', 'gain']}}}, 'required': ['microphone', 'performanceTime']}}]"
|
||
},
|
||
{
|
||
"id": "61cfd720",
|
||
"content": "Conversation history 1..t:\nuser: I have been fascinated recently with total solar eclipses. I am planning my next travel and would like to know when the next total solar eclipse will be visible in Greece, specifically in Athens, over the next five years.\n"
|
||
}
|
||
],
|
||
"target": "{\"ground_truth\": {\"NightSkyAnalysis_performEclipseAnalysis\": {\"dateRange\": {\"startDate\": \"2023-01-01\", \"endDate\": \"2028-01-01\"}, \"location\": {\"latitude\": 37.9838, \"longitude\": 23.7275}, \"eclipseType\": \"total\"}}, \"mile_stone\": []}",
|
||
"id": 0,
|
||
"group_id": 0,
|
||
"subset_key": "normal_single_turn_single_function",
|
||
"metadata": {
|
||
"id": "normal_single_turn_single_function_0",
|
||
"test_category": "normal_single_turn_single_function",
|
||
"dialogue_id": "normal_single_turn_single_function_0",
|
||
"language": "en",
|
||
"functions": [
|
||
{
|
||
"name": "NightSkyAnalysis_performEclipseAnalysis",
|
||
"description": "Analyzes the occurrence of solar eclipses, categorizes them into types, and predicts future occurrences based on historical data and celestial mechanics.",
|
||
"parameters": {
|
||
"type": "object",
|
||
"properties": {
|
||
"dateRange": {
|
||
"description": "The range of dates for which to analyze solar eclipses.",
|
||
"type": "object",
|
||
"properties": {
|
||
"startDate": {
|
||
"description": "The starting date for the analysis in YYYY-MM-DD format.",
|
||
"type": "string"
|
||
},
|
||
"endDate": {
|
||
"description": "The ending date for the analysis in YYYY-MM-DD format.",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"startDate",
|
||
"endDate"
|
||
]
|
||
},
|
||
"location": {
|
||
"description": "Geographical coordinates to focus the eclipse analysis.",
|
||
"type": "object",
|
||
"properties": {
|
||
"latitude": {
|
||
"description": "Latitude of the location.",
|
||
"type": "number",
|
||
"minimum": -90,
|
||
"maximum": 90
|
||
},
|
||
"longitude": {
|
||
"description": "Longitude of the location.",
|
||
"type": "number",
|
||
"minimum": -180,
|
||
"maximum": 180
|
||
}
|
||
},
|
||
"required": [
|
||
"latitude",
|
||
"longitude"
|
||
]
|
||
},
|
||
"eclipseType": {
|
||
"description": "The type of solar eclipse to specifically analyze.",
|
||
"type": "string",
|
||
"enum": [
|
||
"total",
|
||
"annular",
|
||
"partial"
|
||
]
|
||
}
|
||
},
|
||
"required": [
|
||
"dateRange",
|
||
"location"
|
||
]
|
||
}
|
||
},
|
||
{
|
||
"name": "AudioPerformanceOptimizer_optimizeMicrophoneSettings",
|
||
"description": "Optimizes microphone settings for live performances, focusing on dynamic microphones to enhance sound quality and reduce feedback.",
|
||
"parameters": {
|
||
"type": "object",
|
||
"properties": {
|
||
"microphone": {
|
||
"description": "Details of the microphone used.",
|
||
"type": "object",
|
||
"properties": {
|
||
"type": {
|
||
"description": "Type of the microphone.",
|
||
"type": "string",
|
||
"enum": [
|
||
"dynamic",
|
||
"condenser",
|
||
"ribbon"
|
||
]
|
||
},
|
||
"model": {
|
||
"description": "Model of the microphone.",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"type",
|
||
"model"
|
||
]
|
||
},
|
||
"performanceTime": {
|
||
"description": "Scheduled time for the performance.",
|
||
"type": "string",
|
||
"enum": [
|
||
"morning",
|
||
"afternoon",
|
||
"evening",
|
||
"night"
|
||
]
|
||
},
|
||
"environment": {
|
||
"description": "Environmental conditions of the performance area.",
|
||
"type": "object",
|
||
"properties": {
|
||
"humidity": {
|
||
"description": "Humidity level as a percentage.",
|
||
"type": "integer",
|
||
"minimum": 0,
|
||
"maximum": 100
|
||
},
|
||
"temperature": {
|
||
"description": "Temperature in Celsius.",
|
||
"type": "integer"
|
||
}
|
||
}
|
||
},
|
||
"soundSettings": {
|
||
"description": "Specific sound settings to apply.",
|
||
"type": "array",
|
||
"items": {
|
||
"type": "object",
|
||
"properties": {
|
||
"frequency": {
|
||
"description": "Frequency adjustments in Hz.",
|
||
"type": "integer"
|
||
},
|
||
"gain": {
|
||
"description": "Gain adjustments in dB.",
|
||
"type": "integer"
|
||
},
|
||
"effects": {
|
||
"description": "List of audio effects to apply.",
|
||
"type": "array",
|
||
"items": {
|
||
"type": "string",
|
||
"enum": [
|
||
"reverb",
|
||
"echo",
|
||
"distortion"
|
||
]
|
||
}
|
||
}
|
||
},
|
||
"required": [
|
||
"frequency",
|
||
"gain"
|
||
]
|
||
}
|
||
}
|
||
},
|
||
"required": [
|
||
"microphone",
|
||
"performanceTime"
|
||
]
|
||
}
|
||
}
|
||
],
|
||
"ground_truth": {
|
||
"NightSkyAnalysis_performEclipseAnalysis": {
|
||
"dateRange": {
|
||
"startDate": "2023-01-01",
|
||
"endDate": "2028-01-01"
|
||
},
|
||
"location": {
|
||
"latitude": 37.9838,
|
||
"longitude": 23.7275
|
||
},
|
||
"eclipseType": "total"
|
||
}
|
||
},
|
||
"mile_stone": [],
|
||
"initial_config": {},
|
||
"involved_classes": [],
|
||
"question": "user: I have been fascinated recently with total solar eclipses. I am planning my next travel and would like to know when the next total solar eclipse will be visible in Greece, specifically in Athens, over the next five years.\n",
|
||
"time": "The current time is January 01, 2023, Sunday",
|
||
"profile": ""
|
||
}
|
||
},
|
||
"subset": "normal_single_turn_single_function",
|
||
"truncated": true
|
||
},
|
||
"readme": {
|
||
"en": "# ACEBench\n\n\n## Overview\n\nACEBench evaluates whether large language models can use tools in realistic settings: picking the\nright API, filling its arguments, pushing back on requests that cannot be satisfied, and driving\nmulti-step agent tasks against a simulated environment. Data is split into three families -\n`normal` (ordinary tool use), `special` (incomplete, incorrect or out-of-scope requests) and\n`agent` (multi-step and multi-turn interaction) - reported over 17 fine-grained categories.\n\n## Task Description\n\n- **Task Type**: Function calling and agentic tool use\n- **Input**: Conversation history, API specifications, and optional time or character-profile context\n- **Output**: A `[ApiName(key='value')]` call list, a diagnostic sentence, or a full agent trajectory\n- **Domain**: 8 domains and 68 sub-domains including technology, finance, health and society\n\n## Key Features\n\n- 1023 English and 1017 Chinese samples, selectable through `extra_params.language`.\n- Uses the official ACEBench prompts and the official `[ApiName(...)]` output contract, so an\n output that cannot be decoded scores zero instead of being rescued by lenient parsing.\n- `normal_multi_turn_*` categories are scored per dialogue: every step must be correct for the\n dialogue to count, matching the official turn-level aggregation.\n- `agent` categories run a real rollout against ACEBench's simulated phone, food-delivery and\n travel APIs, and are graded on the resulting environment state.\n\n## Evaluation Notes\n\n- `accuracy` is the primary metric. For `normal` and `special` it is answer accuracy; for `agent` it is\n end-state accuracy. `process_acc` additionally reports milestone progress for `agent` samples and\n per-step progress for `normal_multi_turn_*` samples.\n- The report adds the official groupings (ATOM, SINGLE_TURN, MULTI_TURN, NORMAL, SPECIAL, AGENT)\n and an OVERALL score weighted `normal` 0.578 / `special` 0.2676 / `agent` 0.1545. Weights are\n renormalized over the groups actually evaluated, so a partial run stays interpretable.\n- `agent_multi_turn` additionally needs a user simulator; set `extra_params.user_model` to the model\n that should play the user (the official runner uses `gpt-4o`). Without it those rollouts fail and\n score zero, so configure it before reading an OVERALL number.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `acebench` |\n| **Dataset ID** | [evalscope/acebench](https://modelscope.cn/datasets/evalscope/acebench/summary) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `FunctionCalling`, `MultiTurn` |\n| **Metrics** | `accuracy`, `process_acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `normal` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 1,023 |\n| Prompt Length (Mean) | 6032.98 chars |\n| Prompt Length (Min/Max) | 2295 / 11835 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `normal_single_turn_single_function` | 100 | 5165.79 | 2461 | 9553 |\n| `normal_single_turn_parallel_function` | 100 | 5036.21 | 2295 | 9644 |\n| `normal_multi_turn_user_adjust` | 123 | 4658.51 | 3172 | 6976 |\n| `normal_multi_turn_user_switch` | 100 | 7546.46 | 3467 | 11835 |\n| `normal_similar_api` | 50 | 3511.84 | 2484 | 6209 |\n| `normal_preference` | 50 | 8637.66 | 7107 | 10381 |\n| `normal_atom_bool` | 50 | 7377.62 | 4762 | 9727 |\n| `normal_atom_enum` | 50 | 7676.94 | 4927 | 11337 |\n| `normal_atom_number` | 50 | 7481.46 | 4851 | 10278 |\n| `normal_atom_list` | 50 | 7524.06 | 4910 | 10514 |\n| `normal_atom_object_deep` | 50 | 6102.02 | 2873 | 9755 |\n| `normal_atom_object_short` | 50 | 5139.5 | 2343 | 8921 |\n| `special_incomplete` | 50 | 6177.34 | 3473 | 10806 |\n| `special_error_param` | 50 | 4499.78 | 3121 | 6090 |\n| `special_irrelevant` | 50 | 6011.94 | 3778 | 8492 |\n| `agent_multi_step` | 20 | 6407.9 | 6343 | 6472 |\n| `agent_multi_turn` | 30 | 6290.97 | 5505 | 6630 |\n\n## Sample Example\n\n**Subset**: `normal_single_turn_single_function`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"9198db95\",\n \"content\": \"You are an AI assistant with the role name \\\"assistant.\\\" Based on the provided API specifications and conversation history from steps 1 to t, generate the API requests that the assistant should call in step t+1. The API requests should be outp ... [TRUNCATED 3788 chars] ... '}, 'effects': {'description': 'List of audio effects to apply.', 'type': 'array', 'items': {'type': 'string', 'enum': ['reverb', 'echo', 'distortion']}}}, 'required': ['frequency', 'gain']}}}, 'required': ['microphone', 'performanceTime']}}]\"\n },\n {\n \"id\": \"61cfd720\",\n \"content\": \"Conversation history 1..t:\\nuser: I have been fascinated recently with total solar eclipses. I am planning my next travel and would like to know when the next total solar eclipse will be visible in Greece, specifically in Athens, over the next five years.\\n\"\n }\n ],\n \"target\": \"{\\\"ground_truth\\\": {\\\"NightSkyAnalysis_performEclipseAnalysis\\\": {\\\"dateRange\\\": {\\\"startDate\\\": \\\"2023-01-01\\\", \\\"endDate\\\": \\\"2028-01-01\\\"}, \\\"location\\\": {\\\"latitude\\\": 37.9838, \\\"longitude\\\": 23.7275}, \\\"eclipseType\\\": \\\"total\\\"}}, \\\"mile_stone\\\": []}\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"normal_single_turn_single_function\",\n \"metadata\": {\n \"id\": \"normal_single_turn_single_function_0\",\n \"test_category\": \"normal_single_turn_single_function\",\n \"dialogue_id\": \"normal_single_turn_single_function_0\",\n \"language\": \"en\",\n \"functions\": [\n {\n \"name\": \"NightSkyAnalysis_performEclipseAnalysis\",\n \"description\": \"Analyzes the occurrence of solar eclipses, categorizes them into types, and predicts future occurrences based on historical data and celestial mechanics.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"dateRange\": {\n \"description\": \"The range of dates for which to analyze solar eclipses.\",\n \"type\": \"object\",\n \"properties\": {\n \"startDate\": {\n \"description\": \"The starting date for the analysis in YYYY-MM-DD format.\",\n \"type\": \"string\"\n },\n \"endDate\": {\n \"description\": \"The ending date for the analysis in YYYY-MM-DD format.\",\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"startDate\",\n \"endDate\"\n ]\n },\n \"location\": {\n \"description\": \"Geographical coordinates to focus the eclipse analysis.\",\n \"type\": \"object\",\n \"properties\": {\n \"latitude\": {\n \"description\": \"Latitude of the location.\",\n \"type\": \"number\",\n \"minimum\": -90,\n \"maximum\": 90\n },\n \"longitude\": {\n \"description\": \"Longitude of the location.\",\n \"type\": \"number\",\n \"minimum\": -180,\n \"maximum\": 180\n }\n },\n \"required\": [\n \"latitude\",\n \"longitude\"\n ]\n },\n \"eclipseType\": {\n \"description\": \"The type of solar eclipse to specifically analyze.\",\n \"type\": \"string\",\n \"enum\": [\n \"total\",\n \"annular\",\n \"partial\"\n ]\n }\n },\n \"required\": [\n \"dateRange\",\n \"location\"\n ]\n }\n },\n {\n \"name\": \"AudioPerformanceOptimizer_optimizeMicrophoneSettings\",\n \"description\": \"Optimizes microphone settings for live performances, focusing on dynamic microphones to enhance sound quality and reduce feedback.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"microphone\": {\n \"description\": \"Details of the microphone used.\",\n \"type\": \"object\",\n \"properties\": {\n \"type\": {\n \"description\": \"Type of the microphone.\",\n \"type\": \"string\",\n \"enum\": [\n \"dynamic\",\n \"condenser\",\n \"ribbon\"\n ]\n },\n \"model\": {\n \"description\": \"Model of the microphone.\",\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"type\",\n \"model\"\n ]\n },\n \"performanceTime\": {\n \"description\": \"Scheduled time for the performance.\",\n \"type\": \"string\",\n \"enum\": [\n \"morning\",\n \"afternoon\",\n \"evening\",\n \"night\"\n ]\n },\n \"environment\": {\n \"description\": \"Environmental conditions of the performance area.\",\n \"type\": \"object\",\n \"properties\": {\n \"humidity\": {\n \"description\": \"Humidity level as a percentage.\",\n \"type\": \"integer\",\n \"minimum\": 0,\n \"maximum\": 100\n },\n \"temperature\": {\n \"description\": \"Temperature in Celsius.\",\n \"type\": \"integer\"\n }\n }\n },\n \"soundSettings\": {\n \"description\": \"Specific sound settings to apply.\",\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"frequency\": {\n \"description\": \"Frequency adjustments in Hz.\",\n \"type\": \"integer\"\n },\n \"gain\": {\n \"description\": \"Gain adjustments in dB.\",\n \"type\": \"integer\"\n },\n \"effects\": {\n \"description\": \"List of audio effects to apply.\",\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\",\n \"enum\": [\n \"reverb\",\n \"echo\",\n \"distortion\"\n ]\n }\n }\n },\n \"required\": [\n \"frequency\",\n \"gain\"\n ]\n }\n }\n },\n \"required\": [\n \"microphone\",\n \"performanceTime\"\n ]\n }\n }\n ],\n \"ground_truth\": {\n \"NightSkyAnalysis_performEclipseAnalysis\": {\n \"dateRange\": {\n \"startDate\": \"2023-01-01\",\n \"endDate\": \"2028-01-01\"\n },\n \"location\": {\n \"latitude\": 37.9838,\n \"longitude\": 23.7275\n },\n \"eclipseType\": \"total\"\n }\n },\n \"mile_stone\": [],\n \"initial_config\": {},\n \"involved_classes\": [],\n \"question\": \"user: I have been fascinated recently with total solar eclipses. I am planning my next travel and would like to know when the next total solar eclipse will be visible in Greece, specifically in Athens, over the next five years.\\n\",\n \"time\": \"The current time is January 01, 2023, Sunday\",\n \"profile\": \"\"\n }\n}\n```\n\n*Note: Some content was truncated for display.*\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `language` | `str` | `en` | Dataset language to evaluate, either `en` or `zh`. |\n| `user_model` | `str` | `` | Model that plays the user in `agent_multi_turn` rollouts, e.g. `gpt-4o`. Those rollouts fail and score zero when unset. |\n| `user_model_api_url` | `str` | `` | Base URL for `user_model`. Defaults to `MODELSCOPE_API_BASE`. |\n| `user_model_api_key` | `str` | `` | API key for `user_model`. Defaults to `MODELSCOPE_SDK_TOKEN`. |\n| `max_dialog_turns` | `int` | `40` | Maximum number of agent rollout steps. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets acebench \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['acebench'],\n dataset_args={\n 'acebench': {\n # subset_list: ['normal_single_turn_single_function', 'normal_single_turn_parallel_function', 'normal_multi_turn_user_adjust'] # optional, evaluate specific subsets\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n",
|
||
"zh": "# ACEBench\n\n\n## 概述\n\nACEBench 评估大语言模型在真实场景中使用工具的能力：选择正确的 API、填充参数、拒绝无法满足的请求，以及在模拟环境中驱动多步骤智能体任务。数据分为三类——`normal`（常规工具使用）、`special`（不完整、错误或超出范围的请求）和 `agent`（多步骤、多轮交互）——并在 17 个细粒度类别上进行报告。\n\n## 任务描述\n\n- **任务类型**：函数调用与智能体工具使用\n- **输入**：对话历史、API 规范，以及可选的时间或角色档案上下文\n- **输出**：一个 `[ApiName(key='value')]` 调用列表、一句诊断说明，或完整的智能体轨迹\n- **领域**：涵盖技术、金融、健康和社会等 8 个领域及 68 个子领域\n\n## 主要特性\n\n- 包含 1023 个英文样本和 1017 个中文样本，可通过 `extra_params.language` 选择。\n- 使用官方 ACEBench 提示词和官方 `[ApiName(...)]` 输出格式；若输出无法解析，则直接得零分，而非通过宽松解析进行挽救。\n- `normal_multi_turn_*` 类别按对话评分：只有所有步骤均正确，该对话才算正确，符合官方的逐轮聚合方式。\n- `agent` 类别会在 ACEBench 模拟的手机、外卖和旅行 API 上执行真实 rollout，并根据最终环境状态进行评分。\n\n## 评估说明\n\n- `accuracy` 是主要指标。对于 `normal` 和 `special` 类别，指答案准确率；对于 `agent` 类别，指最终状态准确率。`process_acc` 额外报告 `agent` 样本的关键里程碑进展，以及 `normal_multi_turn_*` 样本的逐轮进展。\n- 报告包含官方分组（ATOM、SINGLE_TURN、MULTI_TURN、NORMAL、SPECIAL、AGENT）和一个加权的 OVERALL 分数，权重为 `normal` 0.578 / `special` 0.2676 / `agent` 0.1545。权重会根据实际评估的分组重新归一化，因此部分运行结果仍具可解释性。\n- `agent_multi_turn` 还需要一个用户模拟器；请通过 `extra_params.user_model` 设置扮演用户的模型（官方运行器使用 `gpt-4o`）。若未设置，这些 rollout 将失败并得零分，因此在查看 OVERALL 分数前请务必配置此项。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `acebench` |\n| **数据集ID** | [evalscope/acebench](https://modelscope.cn/datasets/evalscope/acebench/summary) |\n| **论文** | N/A |\n| **标签** | `Agent`, `FunctionCalling`, `MultiTurn` |\n| **指标** | `accuracy`, `process_acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `normal` |\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 1,023 |\n| 提示词长度（平均） | 6032.98 字符 |\n| 提示词长度（最小/最大） | 2295 / 11835 字符 |\n\n**各子集统计数据：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `normal_single_turn_single_function` | 100 | 5165.79 | 2461 | 9553 |\n| `normal_single_turn_parallel_function` | 100 | 5036.21 | 2295 | 9644 |\n| `normal_multi_turn_user_adjust` | 123 | 4658.51 | 3172 | 6976 |\n| `normal_multi_turn_user_switch` | 100 | 7546.46 | 3467 | 11835 |\n| `normal_similar_api` | 50 | 3511.84 | 2484 | 6209 |\n| `normal_preference` | 50 | 8637.66 | 7107 | 10381 |\n| `normal_atom_bool` | 50 | 7377.62 | 4762 | 9727 |\n| `normal_atom_enum` | 50 | 7676.94 | 4927 | 11337 |\n| `normal_atom_number` | 50 | 7481.46 | 4851 | 10278 |\n| `normal_atom_list` | 50 | 7524.06 | 4910 | 10514 |\n| `normal_atom_object_deep` | 50 | 6102.02 | 2873 | 9755 |\n| `normal_atom_object_short` | 50 | 5139.5 | 2343 | 8921 |\n| `special_incomplete` | 50 | 6177.34 | 3473 | 10806 |\n| `special_error_param` | 50 | 4499.78 | 3121 | 6090 |\n| `special_irrelevant` | 50 | 6011.94 | 3778 | 8492 |\n| `agent_multi_step` | 20 | 6407.9 | 6343 | 6472 |\n| `agent_multi_turn` | 30 | 6290.97 | 5505 | 6630 |\n\n## 样例示例\n\n**子集**: `normal_single_turn_single_function`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"9198db95\",\n \"content\": \"You are an AI assistant with the role name \\\"assistant.\\\" Based on the provided API specifications and conversation history from steps 1 to t, generate the API requests that the assistant should call in step t+1. The API requests should be outp ... [TRUNCATED 3788 chars] ... '}, 'effects': {'description': 'List of audio effects to apply.', 'type': 'array', 'items': {'type': 'string', 'enum': ['reverb', 'echo', 'distortion']}}}, 'required': ['frequency', 'gain']}}}, 'required': ['microphone', 'performanceTime']}}]\"\n },\n {\n \"id\": \"61cfd720\",\n \"content\": \"Conversation history 1..t:\\nuser: I have been fascinated recently with total solar eclipses. I am planning my next travel and would like to know when the next total solar eclipse will be visible in Greece, specifically in Athens, over the next five years.\\n\"\n }\n ],\n \"target\": \"{\\\"ground_truth\\\": {\\\"NightSkyAnalysis_performEclipseAnalysis\\\": {\\\"dateRange\\\": {\\\"startDate\\\": \\\"2023-01-01\\\", \\\"endDate\\\": \\\"2028-01-01\\\"}, \\\"location\\\": {\\\"latitude\\\": 37.9838, \\\"longitude\\\": 23.7275}, \\\"eclipseType\\\": \\\"total\\\"}}, \\\"mile_stone\\\": []}\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"normal_single_turn_single_function\",\n \"metadata\": {\n \"id\": \"normal_single_turn_single_function_0\",\n \"test_category\": \"normal_single_turn_single_function\",\n \"dialogue_id\": \"normal_single_turn_single_function_0\",\n \"language\": \"en\",\n \"functions\": [\n {\n \"name\": \"NightSkyAnalysis_performEclipseAnalysis\",\n \"description\": \"Analyzes the occurrence of solar eclipses, categorizes them into types, and predicts future occurrences based on historical data and celestial mechanics.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"dateRange\": {\n \"description\": \"The range of dates for which to analyze solar eclipses.\",\n \"type\": \"object\",\n \"properties\": {\n \"startDate\": {\n \"description\": \"The starting date for the analysis in YYYY-MM-DD format.\",\n \"type\": \"string\"\n },\n \"endDate\": {\n \"description\": \"The ending date for the analysis in YYYY-MM-DD format.\",\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"startDate\",\n \"endDate\"\n ]\n },\n \"location\": {\n \"description\": \"Geographical coordinates to focus the eclipse analysis.\",\n \"type\": \"object\",\n \"properties\": {\n \"latitude\": {\n \"description\": \"Latitude of the location.\",\n \"type\": \"number\",\n \"minimum\": -90,\n \"maximum\": 90\n },\n \"longitude\": {\n \"description\": \"Longitude of the location.\",\n \"type\": \"number\",\n \"minimum\": -180,\n \"maximum\": 180\n }\n },\n \"required\": [\n \"latitude\",\n \"longitude\"\n ]\n },\n \"eclipseType\": {\n \"description\": \"The type of solar eclipse to specifically analyze.\",\n \"type\": \"string\",\n \"enum\": [\n \"total\",\n \"annular\",\n \"partial\"\n ]\n }\n },\n \"required\": [\n \"dateRange\",\n \"location\"\n ]\n }\n },\n {\n \"name\": \"AudioPerformanceOptimizer_optimizeMicrophoneSettings\",\n \"description\": \"Optimizes microphone settings for live performances, focusing on dynamic microphones to enhance sound quality and reduce feedback.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"microphone\": {\n \"description\": \"Details of the microphone used.\",\n \"type\": \"object\",\n \"properties\": {\n \"type\": {\n \"description\": \"Type of the microphone.\",\n \"type\": \"string\",\n \"enum\": [\n \"dynamic\",\n \"condenser\",\n \"ribbon\"\n ]\n },\n \"model\": {\n \"description\": \"Model of the microphone.\",\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"type\",\n \"model\"\n ]\n },\n \"performanceTime\": {\n \"description\": \"Scheduled time for the performance.\",\n \"type\": \"string\",\n \"enum\": [\n \"morning\",\n \"afternoon\",\n \"evening\",\n \"night\"\n ]\n },\n \"environment\": {\n \"description\": \"Environmental conditions of the performance area.\",\n \"type\": \"object\",\n \"properties\": {\n \"humidity\": {\n \"description\": \"Humidity level as a percentage.\",\n \"type\": \"integer\",\n \"minimum\": 0,\n \"maximum\": 100\n },\n \"temperature\": {\n \"description\": \"Temperature in Celsius.\",\n \"type\": \"integer\"\n }\n }\n },\n \"soundSettings\": {\n \"description\": \"Specific sound settings to apply.\",\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"frequency\": {\n \"description\": \"Frequency adjustments in Hz.\",\n \"type\": \"integer\"\n },\n \"gain\": {\n \"description\": \"Gain adjustments in dB.\",\n \"type\": \"integer\"\n },\n \"effects\": {\n \"description\": \"List of audio effects to apply.\",\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\",\n \"enum\": [\n \"reverb\",\n \"echo\",\n \"distortion\"\n ]\n }\n }\n },\n \"required\": [\n \"frequency\",\n \"gain\"\n ]\n }\n }\n },\n \"required\": [\n \"microphone\",\n \"performanceTime\"\n ]\n }\n }\n ],\n \"ground_truth\": {\n \"NightSkyAnalysis_performEclipseAnalysis\": {\n \"dateRange\": {\n \"startDate\": \"2023-01-01\",\n \"endDate\": \"2028-01-01\"\n },\n \"location\": {\n \"latitude\": 37.9838,\n \"longitude\": 23.7275\n },\n \"eclipseType\": \"total\"\n }\n },\n \"mile_stone\": [],\n \"initial_config\": {},\n \"involved_classes\": [],\n \"question\": \"user: I have been fascinated recently with total solar eclipses. I am planning my next travel and would like to know when the next total solar eclipse will be visible in Greece, specifically in Athens, over the next five years.\\n\",\n \"time\": \"The current time is January 01, 2023, Sunday\",\n \"profile\": \"\"\n }\n}\n```\n\n*注：部分内容因展示需要已被截断。*\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `language` | `str` | `en` | 要评估的数据集语言，可选 `en` 或 `zh`。 |\n| `user_model` | `str` | `` | 在 `agent_multi_turn` rollout 中扮演用户的模型，例如 `gpt-4o`。若未设置，这些 rollout 将失败并得分为零。 |\n| `user_model_api_url` | `str` | `` | `user_model` 的基础 URL，默认为 `MODELSCOPE_API_BASE`。 |\n| `user_model_api_key` | `str` | `` | `user_model` 的 API 密钥，默认为 `MODELSCOPE_SDK_TOKEN`。 |\n| `max_dialog_turns` | `int` | `40` | 智能体 rollout 的最大步数。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets acebench \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['acebench'],\n dataset_args={\n 'acebench': {\n # subset_list: ['normal_single_turn_single_function', 'normal_single_turn_parallel_function', 'normal_multi_turn_user_adjust'] # 可选，评估特定子集\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```",
|
||
"content_hash": "7264b099ada01ab9b002e7dd9313af10",
|
||
"needs_translation": false
|
||
},
|
||
"updated_at": "2026-08-11T16:13:45.169873",
|
||
"translation_updated_at": "2026-08-05T14:52:58"
|
||
}
|