184 lines
6.5 KiB
JSON
184 lines
6.5 KiB
JSON
{
|
|
"name": "qwen3-max@ifeval",
|
|
"dataset_name": "ifeval",
|
|
"dataset_pretty_name": "IFEval",
|
|
"dataset_description": "\n## Overview\n\nIFEval (Instruction-Following Eval) is a benchmark for evaluating how well language models follow explicit, verifiable instructions. It contains prompts with specific formatting, content, or structural requirements that can be objectively verified.\n\n## Task Description\n\n- **Task Type**: Instruction Following Evaluation\n- **Input**: Prompts with explicit, verifiable constraints\n- **Output**: Response that follows all specified instructions\n- **Constraint Types**: Format, length, keywords, structure, etc.\n\n## Key Features\n\n- ~500 prompts with 25 types of verifiable instructions\n- Instructions are objectively checkable (not subjective)\n- Examples: \"write exactly 3 paragraphs\", \"include the word X\", \"use bullet points\"\n- Tests instruction comprehension and compliance\n- No ambiguity in evaluation criteria\n\n## Evaluation Notes\n\n- Default configuration uses **0-shot** evaluation\n- Four metrics available:\n - `prompt_level_strict`: All instructions in prompt must be followed\n - `prompt_level_loose`: Some tolerance for minor deviations\n - `inst_level_strict`: Per-instruction accuracy (strict)\n - `inst_level_loose`: Per-instruction accuracy (loose)\n- `prompt_level_strict` is the primary metric\n- Automatic verification of instruction compliance\n",
|
|
"model_name": "qwen3-max",
|
|
"score": 0.8,
|
|
"metrics": [
|
|
{
|
|
"name": "mean_prompt_level_strict",
|
|
"num": 10,
|
|
"score": 0.8,
|
|
"macro_score": 0.8,
|
|
"categories": [
|
|
{
|
|
"name": [
|
|
"default"
|
|
],
|
|
"num": 10,
|
|
"score": 0.8,
|
|
"macro_score": 0.8,
|
|
"subsets": [
|
|
{
|
|
"name": "default",
|
|
"score": 0.8,
|
|
"num": 10
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "mean_inst_level_strict",
|
|
"num": 10,
|
|
"score": 0.8667,
|
|
"macro_score": 0.8667,
|
|
"categories": [
|
|
{
|
|
"name": [
|
|
"default"
|
|
],
|
|
"num": 10,
|
|
"score": 0.8667,
|
|
"macro_score": 0.8667,
|
|
"subsets": [
|
|
{
|
|
"name": "default",
|
|
"score": 0.8667,
|
|
"num": 10
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "mean_prompt_level_loose",
|
|
"num": 10,
|
|
"score": 0.9,
|
|
"macro_score": 0.9,
|
|
"categories": [
|
|
{
|
|
"name": [
|
|
"default"
|
|
],
|
|
"num": 10,
|
|
"score": 0.9,
|
|
"macro_score": 0.9,
|
|
"subsets": [
|
|
{
|
|
"name": "default",
|
|
"score": 0.9,
|
|
"num": 10
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "mean_inst_level_loose",
|
|
"num": 10,
|
|
"score": 0.9667,
|
|
"macro_score": 0.9667,
|
|
"categories": [
|
|
{
|
|
"name": [
|
|
"default"
|
|
],
|
|
"num": 10,
|
|
"score": 0.9667,
|
|
"macro_score": 0.9667,
|
|
"subsets": [
|
|
{
|
|
"name": "default",
|
|
"score": 0.9667,
|
|
"num": 10
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"analysis": "N/A",
|
|
"perf_metrics": {
|
|
"summary": {
|
|
"n_samples": 10,
|
|
"latency": {
|
|
"mean": 11.014216,
|
|
"std": 6.164133,
|
|
"min": 0.860077,
|
|
"25%": 5.774487,
|
|
"50%": 12.656771,
|
|
"75%": 14.669187,
|
|
"90%": 18.490001,
|
|
"99%": 19.207115,
|
|
"max": 19.286794
|
|
},
|
|
"throughput": {
|
|
"avg_output_tps": 31.79,
|
|
"avg_req_ps": 0.0908
|
|
},
|
|
"usage": {
|
|
"input_tokens": {
|
|
"mean": 57.5,
|
|
"std": 19.648297,
|
|
"min": 25.0,
|
|
"25%": 47.25,
|
|
"50%": 52.0,
|
|
"75%": 74.5,
|
|
"90%": 84.1,
|
|
"99%": 84.91,
|
|
"max": 85.0
|
|
},
|
|
"output_tokens": {
|
|
"mean": 350.1,
|
|
"std": 202.890474,
|
|
"min": 10.0,
|
|
"25%": 186.0,
|
|
"50%": 393.0,
|
|
"75%": 475.75,
|
|
"90%": 602.0,
|
|
"99%": 618.2,
|
|
"max": 620.0
|
|
},
|
|
"total_tokens": {
|
|
"mean": 407.6,
|
|
"std": 204.721057,
|
|
"min": 58.0,
|
|
"25%": 267.75,
|
|
"50%": 441.0,
|
|
"75%": 553.0,
|
|
"90%": 649.8,
|
|
"99%": 672.48,
|
|
"max": 675.0
|
|
},
|
|
"total_input_tokens": 575,
|
|
"total_output_tokens": 3501,
|
|
"total_tokens_count": 4076
|
|
},
|
|
"ttft": {
|
|
"mean": 0.556846,
|
|
"std": 0.13523,
|
|
"min": 0.425786,
|
|
"25%": 0.492836,
|
|
"50%": 0.533555,
|
|
"75%": 0.557139,
|
|
"90%": 0.663034,
|
|
"99%": 0.878001,
|
|
"max": 0.901887
|
|
},
|
|
"tpot": {
|
|
"mean": 0.030132,
|
|
"std": 0.001378,
|
|
"min": 0.028593,
|
|
"25%": 0.029462,
|
|
"50%": 0.029738,
|
|
"75%": 0.029854,
|
|
"90%": 0.032247,
|
|
"99%": 0.032995,
|
|
"max": 0.033078
|
|
}
|
|
}
|
|
},
|
|
"num": 10
|
|
} |