{
    "name": "qwen3-max@ifeval",
    "dataset_name": "ifeval",
    "dataset_pretty_name": "IFEval",
    "dataset_description": "\n## Overview\n\nIFEval (Instruction-Following Eval) is a benchmark for evaluating how well language models follow explicit, verifiable instructions. It contains prompts with specific formatting, content, or structural requirements that can be objectively verified.\n\n## Task Description\n\n- **Task Type**: Instruction Following Evaluation\n- **Input**: Prompts with explicit, verifiable constraints\n- **Output**: Response that follows all specified instructions\n- **Constraint Types**: Format, length, keywords, structure, etc.\n\n## Key Features\n\n- ~500 prompts with 25 types of verifiable instructions\n- Instructions are objectively checkable (not subjective)\n- Examples: \"write exactly 3 paragraphs\", \"include the word X\", \"use bullet points\"\n- Tests instruction comprehension and compliance\n- No ambiguity in evaluation criteria\n\n## Evaluation Notes\n\n- Default configuration uses **0-shot** evaluation\n- Four metrics available:\n  - `prompt_level_strict`: All instructions in prompt must be followed\n  - `prompt_level_loose`: Some tolerance for minor deviations\n  - `inst_level_strict`: Per-instruction accuracy (strict)\n  - `inst_level_loose`: Per-instruction accuracy (loose)\n- `prompt_level_strict` is the primary metric\n- Automatic verification of instruction compliance\n",
    "model_name": "qwen3-max",
    "score": 0.8,
    "metrics": [
        {
            "name": "mean_prompt_level_strict",
            "num": 10,
            "score": 0.8,
            "macro_score": 0.8,
            "categories": [
                {
                    "name": [
                        "default"
                    ],
                    "num": 10,
                    "score": 0.8,
                    "macro_score": 0.8,
                    "subsets": [
                        {
                            "name": "default",
                            "score": 0.8,
                            "num": 10
                        }
                    ]
                }
            ]
        },
        {
            "name": "mean_inst_level_strict",
            "num": 10,
            "score": 0.8667,
            "macro_score": 0.8667,
            "categories": [
                {
                    "name": [
                        "default"
                    ],
                    "num": 10,
                    "score": 0.8667,
                    "macro_score": 0.8667,
                    "subsets": [
                        {
                            "name": "default",
                            "score": 0.8667,
                            "num": 10
                        }
                    ]
                }
            ]
        },
        {
            "name": "mean_prompt_level_loose",
            "num": 10,
            "score": 0.9,
            "macro_score": 0.9,
            "categories": [
                {
                    "name": [
                        "default"
                    ],
                    "num": 10,
                    "score": 0.9,
                    "macro_score": 0.9,
                    "subsets": [
                        {
                            "name": "default",
                            "score": 0.9,
                            "num": 10
                        }
                    ]
                }
            ]
        },
        {
            "name": "mean_inst_level_loose",
            "num": 10,
            "score": 0.9667,
            "macro_score": 0.9667,
            "categories": [
                {
                    "name": [
                        "default"
                    ],
                    "num": 10,
                    "score": 0.9667,
                    "macro_score": 0.9667,
                    "subsets": [
                        {
                            "name": "default",
                            "score": 0.9667,
                            "num": 10
                        }
                    ]
                }
            ]
        }
    ],
    "analysis": "N/A",
    "perf_metrics": {
        "summary": {
            "n_samples": 10,
            "latency": {
                "mean": 11.014216,
                "std": 6.164133,
                "min": 0.860077,
                "25%": 5.774487,
                "50%": 12.656771,
                "75%": 14.669187,
                "90%": 18.490001,
                "99%": 19.207115,
                "max": 19.286794
            },
            "throughput": {
                "avg_output_tps": 31.79,
                "avg_req_ps": 0.0908
            },
            "usage": {
                "input_tokens": {
                    "mean": 57.5,
                    "std": 19.648297,
                    "min": 25.0,
                    "25%": 47.25,
                    "50%": 52.0,
                    "75%": 74.5,
                    "90%": 84.1,
                    "99%": 84.91,
                    "max": 85.0
                },
                "output_tokens": {
                    "mean": 350.1,
                    "std": 202.890474,
                    "min": 10.0,
                    "25%": 186.0,
                    "50%": 393.0,
                    "75%": 475.75,
                    "90%": 602.0,
                    "99%": 618.2,
                    "max": 620.0
                },
                "total_tokens": {
                    "mean": 407.6,
                    "std": 204.721057,
                    "min": 58.0,
                    "25%": 267.75,
                    "50%": 441.0,
                    "75%": 553.0,
                    "90%": 649.8,
                    "99%": 672.48,
                    "max": 675.0
                },
                "total_input_tokens": 575,
                "total_output_tokens": 3501,
                "total_tokens_count": 4076
            },
            "ttft": {
                "mean": 0.556846,
                "std": 0.13523,
                "min": 0.425786,
                "25%": 0.492836,
                "50%": 0.533555,
                "75%": 0.557139,
                "90%": 0.663034,
                "99%": 0.878001,
                "max": 0.901887
            },
            "tpot": {
                "mean": 0.030132,
                "std": 0.001378,
                "min": 0.028593,
                "25%": 0.029462,
                "50%": 0.029738,
                "75%": 0.029854,
                "90%": 0.032247,
                "99%": 0.032995,
                "max": 0.033078
            }
        }
    },
    "num": 10
}