From 13274243a0514ed61dad89dd1a3c18b0a0b5b841 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Wed, 2 Sep 2026 07:30:48 +0000 Subject: [PATCH] Bump vendored EvalScope and add K3-ready DPV4 configs. Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor --- bash/collect_results.py | 84 +- bash/run.py | 43 +- config/dpv4-int8_nothinking.yaml | 135 +- config/dpv4-int8_thinking.yaml | 138 + evalscope/.github/workflows/ci_test_full.yaml | 2 + evalscope/.github/workflows/code-review.yaml | 39 + evalscope/.github/workflows/frontend.yml | 71 + evalscope/.github/workflows/lint.yml | 4 +- evalscope/.github/workflows/publish.yaml | 14 +- evalscope/.gitignore | 11 +- evalscope/.pre-commit-config.yaml | 51 +- evalscope/AGENTS.md | 56 +- evalscope/CONTRIBUTING.md | 31 +- evalscope/DESIGN.md | 395 ++- evalscope/MANIFEST.in | 9 +- evalscope/Makefile | 34 +- evalscope/README.md | 516 +++- evalscope/README_zh.md | 37 +- .../multimodal/vqa/example_placeholder.jsonl | 5 + .../docs/en/advanced_guides/add_benchmark.md | 142 +- .../en/advanced_guides/custom_dataset/clip.md | 2 +- .../custom_dataset/embedding.md | 2 +- .../en/advanced_guides/custom_dataset/llm.md | 87 +- .../en/advanced_guides/custom_dataset/vlm.md | 174 +- evalscope/docs/en/advanced_guides/sandbox.md | 14 +- evalscope/docs/en/benchmarks/a_okvqa.md | 4 +- evalscope/docs/en/benchmarks/aa_lcr.md | 4 +- evalscope/docs/en/benchmarks/acebench.md | 258 +- evalscope/docs/en/benchmarks/agieval.md | 153 + evalscope/docs/en/benchmarks/ai2d.md | 4 +- evalscope/docs/en/benchmarks/aime24.md | 4 +- evalscope/docs/en/benchmarks/aime25.md | 4 +- evalscope/docs/en/benchmarks/aime26.md | 4 +- .../docs/en/benchmarks/air_bench_chat.md | 41 +- .../en/benchmarks/air_bench_foundation.md | 4 +- evalscope/docs/en/benchmarks/alpaca_eval.md | 4 +- evalscope/docs/en/benchmarks/amc.md | 4 +- evalscope/docs/en/benchmarks/anat_em.md | 4 +- evalscope/docs/en/benchmarks/arc.md | 4 +- evalscope/docs/en/benchmarks/arc_agi_2.md | 115 + evalscope/docs/en/benchmarks/arc_indic.md | 134 + evalscope/docs/en/benchmarks/arena_hard.md | 4 +- evalscope/docs/en/benchmarks/arxivmath.md | 4 +- .../docs/en/benchmarks/arxivrollbench.md | 4 +- .../docs/en/benchmarks/arxivrollbench_full.md | 4 +- .../docs/en/benchmarks/automation_bench.md | 144 + evalscope/docs/en/benchmarks/baby_vision.md | 40 +- evalscope/docs/en/benchmarks/bbh.md | 6 +- evalscope/docs/en/benchmarks/bc2gm.md | 4 +- evalscope/docs/en/benchmarks/bc4chemd.md | 4 +- evalscope/docs/en/benchmarks/bc5cdr.md | 4 +- evalscope/docs/en/benchmarks/bfcl_v3.md | 4 +- evalscope/docs/en/benchmarks/bfcl_v4.md | 4 +- .../en/benchmarks/bhasha_bench_multi_ayur.md | 155 + .../benchmarks/bhasha_bench_multi_finance.md | 95 + .../benchmarks/bhasha_bench_multi_krishi.md | 155 + .../en/benchmarks/bhasha_bench_multi_legal.md | 155 + .../docs/en/benchmarks/bhashabenchv1_ayur.md | 133 + .../en/benchmarks/bhashabenchv1_finance.md | 133 + .../en/benchmarks/bhashabenchv1_krishi.md | 133 + .../docs/en/benchmarks/bhashabenchv1_legal.md | 133 + evalscope/docs/en/benchmarks/bigcodebench.md | 7 +- .../docs/en/benchmarks/bigcodebench_hard.md | 7 +- evalscope/docs/en/benchmarks/biomix_qa.md | 4 +- evalscope/docs/en/benchmarks/blink.md | 4 +- .../en/benchmarks/broad_twitter_corpus.md | 4 +- evalscope/docs/en/benchmarks/browsecomp.md | 2 - evalscope/docs/en/benchmarks/cc_bench.md | 4 +- evalscope/docs/en/benchmarks/cc_ocr_v2.md | 169 ++ evalscope/docs/en/benchmarks/ceval.md | 4 +- evalscope/docs/en/benchmarks/chartqa.md | 2 - evalscope/docs/en/benchmarks/charxiv.md | 36 +- .../docs/en/benchmarks/chinese_simpleqa.md | 2 - evalscope/docs/en/benchmarks/cl_bench.md | 4 +- evalscope/docs/en/benchmarks/claw_eval.md | 141 + evalscope/docs/en/benchmarks/cmath.md | 4 +- evalscope/docs/en/benchmarks/cmmlu.md | 26 +- evalscope/docs/en/benchmarks/cmmmu.md | 4 +- evalscope/docs/en/benchmarks/cmmu.md | 4 +- evalscope/docs/en/benchmarks/coin_flip.md | 9 +- .../docs/en/benchmarks/common_voice_15.md | 2 - .../docs/en/benchmarks/commonsense_qa.md | 4 +- .../docs/en/benchmarks/competition_math.md | 4 +- evalscope/docs/en/benchmarks/conll2003.md | 4 +- evalscope/docs/en/benchmarks/conllpp.md | 4 +- evalscope/docs/en/benchmarks/copious.md | 4 +- evalscope/docs/en/benchmarks/count_qa.md | 103 + evalscope/docs/en/benchmarks/cross_ner.md | 4 +- .../docs/en/benchmarks/data_collection.md | 4 +- evalscope/docs/en/benchmarks/deep_swe.md | 127 + evalscope/docs/en/benchmarks/deepsearchqa.md | 101 + evalscope/docs/en/benchmarks/docmath.md | 4 +- evalscope/docs/en/benchmarks/docvqa.md | 2 - evalscope/docs/en/benchmarks/drivel_binary.md | 4 +- .../docs/en/benchmarks/drivel_multilabel.md | 2 - .../docs/en/benchmarks/drivel_selection.md | 4 +- .../docs/en/benchmarks/drivel_writing.md | 4 +- evalscope/docs/en/benchmarks/drop.md | 4 +- .../docs/en/benchmarks/emb_spatial_bench.md | 149 + evalscope/docs/en/benchmarks/eq_bench.md | 2 - evalscope/docs/en/benchmarks/erqa.md | 4 +- evalscope/docs/en/benchmarks/evalmuse.md | 2 - evalscope/docs/en/benchmarks/fin_ner.md | 4 +- evalscope/docs/en/benchmarks/fleurs.md | 2 - evalscope/docs/en/benchmarks/frames.md | 4 +- evalscope/docs/en/benchmarks/gaia.md | 29 +- evalscope/docs/en/benchmarks/gdpval.md | 21 +- evalscope/docs/en/benchmarks/gedit.md | 10 +- evalscope/docs/en/benchmarks/genai_bench.md | 2 - evalscope/docs/en/benchmarks/general_arena.md | 4 +- evalscope/docs/en/benchmarks/general_fc.md | 2 - evalscope/docs/en/benchmarks/general_mcq.md | 4 +- evalscope/docs/en/benchmarks/general_qa.md | 2 - evalscope/docs/en/benchmarks/general_t2i.md | 2 - evalscope/docs/en/benchmarks/general_vmcq.md | 16 +- evalscope/docs/en/benchmarks/general_vqa.md | 9 +- evalscope/docs/en/benchmarks/genia_ner.md | 4 +- evalscope/docs/en/benchmarks/gpqa_diamond.md | 4 +- evalscope/docs/en/benchmarks/gsm8k.md | 4 +- evalscope/docs/en/benchmarks/gsm8k_indic.md | 134 + evalscope/docs/en/benchmarks/gsm8k_v.md | 4 +- .../docs/en/benchmarks/hallusion_bench.md | 3 - evalscope/docs/en/benchmarks/halueval.md | 4 +- evalscope/docs/en/benchmarks/harvey_ner.md | 4 +- evalscope/docs/en/benchmarks/health_bench.md | 2 - evalscope/docs/en/benchmarks/hellaswag.md | 4 +- evalscope/docs/en/benchmarks/hellaswag_hi.md | 119 + evalscope/docs/en/benchmarks/hipho.md | 183 ++ evalscope/docs/en/benchmarks/hle.md | 4 +- evalscope/docs/en/benchmarks/hmmt25.md | 4 +- evalscope/docs/en/benchmarks/hmmt26.md | 4 +- evalscope/docs/en/benchmarks/hmmt_nov25.md | 114 + evalscope/docs/en/benchmarks/hpdv2.md | 2 - evalscope/docs/en/benchmarks/humaneval.md | 4 +- .../docs/en/benchmarks/humaneval_plus.md | 4 +- evalscope/docs/en/benchmarks/ifbench.md | 4 +- evalscope/docs/en/benchmarks/ifeval.md | 5 +- .../docs/en/benchmarks/imo_answerbench.md | 4 +- evalscope/docs/en/benchmarks/indic_boolq.md | 135 + evalscope/docs/en/benchmarks/indic_param.md | 145 + evalscope/docs/en/benchmarks/infovqa.md | 2 - evalscope/docs/en/benchmarks/iquiz.md | 4 +- evalscope/docs/en/benchmarks/jnlpba.md | 4 +- evalscope/docs/en/benchmarks/jnlpba_rare.md | 4 +- evalscope/docs/en/benchmarks/job_bench.md | 158 + evalscope/docs/en/benchmarks/k2_verifier.md | 2 - evalscope/docs/en/benchmarks/kimi_verifier.md | 6 +- evalscope/docs/en/benchmarks/kina.md | 126 + evalscope/docs/en/benchmarks/librispeech.md | 2 - .../docs/en/benchmarks/live_code_bench.md | 10 +- evalscope/docs/en/benchmarks/locomo.md | 2 - evalscope/docs/en/benchmarks/logi_qa.md | 4 +- evalscope/docs/en/benchmarks/longbench_v2.md | 4 +- evalscope/docs/en/benchmarks/longmemeval.md | 4 +- .../docs/en/benchmarks/maritime_bench.md | 4 +- .../docs/en/benchmarks/maritime_ocr_bench.md | 4 +- evalscope/docs/en/benchmarks/math_500.md | 4 +- evalscope/docs/en/benchmarks/math_qa.md | 4 +- evalscope/docs/en/benchmarks/math_verse.md | 4 +- evalscope/docs/en/benchmarks/math_vision.md | 4 +- evalscope/docs/en/benchmarks/math_vista.md | 4 +- evalscope/docs/en/benchmarks/mbpp.md | 26 +- evalscope/docs/en/benchmarks/mbpp_plus.md | 4 +- evalscope/docs/en/benchmarks/mcp_atlas.md | 14 +- evalscope/docs/en/benchmarks/measure_bench.md | 138 + evalscope/docs/en/benchmarks/med_mcqa.md | 4 +- evalscope/docs/en/benchmarks/medxpertqa.md | 169 ++ evalscope/docs/en/benchmarks/mgsm.md | 4 +- evalscope/docs/en/benchmarks/mia_bench.md | 12 +- evalscope/docs/en/benchmarks/micro_vqa.md | 4 +- evalscope/docs/en/benchmarks/milu.md | 144 + evalscope/docs/en/benchmarks/minerva_math.md | 4 +- .../docs/en/benchmarks/minimax_verifier.md | 2 - evalscope/docs/en/benchmarks/miniwob.md | 135 + .../docs/en/benchmarks/mit_movie_trivia.md | 4 +- .../docs/en/benchmarks/mit_restaurant.md | 4 +- evalscope/docs/en/benchmarks/mm_bench.md | 4 +- evalscope/docs/en/benchmarks/mm_star.md | 4 +- evalscope/docs/en/benchmarks/mmau.md | 4 +- evalscope/docs/en/benchmarks/mmlu.md | 4 +- evalscope/docs/en/benchmarks/mmlu_pro.md | 4 +- evalscope/docs/en/benchmarks/mmlu_redux.md | 4 +- evalscope/docs/en/benchmarks/mmmlu.md | 4 +- evalscope/docs/en/benchmarks/mmmu.md | 4 +- evalscope/docs/en/benchmarks/mmmu_pro.md | 4 +- evalscope/docs/en/benchmarks/mri_mcqa.md | 4 +- evalscope/docs/en/benchmarks/msr_vtt.md | 9 +- evalscope/docs/en/benchmarks/msvd.md | 9 +- evalscope/docs/en/benchmarks/multi_if.md | 5 +- evalscope/docs/en/benchmarks/multi_nerd.md | 4 +- .../docs/en/benchmarks/multiple_humaneval.md | 4 +- evalscope/docs/en/benchmarks/multiple_mbpp.md | 4 +- evalscope/docs/en/benchmarks/music_trivia.md | 4 +- evalscope/docs/en/benchmarks/musr.md | 4 +- evalscope/docs/en/benchmarks/mvbench.md | 15 +- evalscope/docs/en/benchmarks/ncbi.md | 4 +- .../docs/en/benchmarks/needle_haystack.md | 4 +- evalscope/docs/en/benchmarks/ocr_bench.md | 4 +- evalscope/docs/en/benchmarks/ocr_bench_v2.md | 4 +- evalscope/docs/en/benchmarks/officeqa.md | 115 + evalscope/docs/en/benchmarks/olmocr_bench.md | 204 ++ .../docs/en/benchmarks/olympiad_bench.md | 4 +- evalscope/docs/en/benchmarks/omni_bench.md | 4 +- .../docs/en/benchmarks/omni_doc_bench.md | 486 +--- .../docs/en/benchmarks/omni_doc_bench_v1_6.md | 179 ++ .../docs/en/benchmarks/one_million_bench.md | 162 ++ evalscope/docs/en/benchmarks/ontonotes5.md | 4 +- evalscope/docs/en/benchmarks/openai_mrcr.md | 2 - .../docs/en/benchmarks/perception_bench.md | 161 + .../perspective_gap_prompt_writing.md | 157 + .../perspective_gap_role_assignment.md | 157 + evalscope/docs/en/benchmarks/phyx_mc.md | 159 + evalscope/docs/en/benchmarks/phyx_oe.md | 161 + evalscope/docs/en/benchmarks/piqa.md | 4 +- evalscope/docs/en/benchmarks/plawbench.md | 162 ++ evalscope/docs/en/benchmarks/pmc_vqa.md | 146 + evalscope/docs/en/benchmarks/poly_math.md | 4 +- evalscope/docs/en/benchmarks/pope.md | 11 +- evalscope/docs/en/benchmarks/prbench.md | 240 ++ evalscope/docs/en/benchmarks/process_bench.md | 2 - evalscope/docs/en/benchmarks/pubmedqa.md | 4 +- evalscope/docs/en/benchmarks/qasc.md | 4 +- evalscope/docs/en/benchmarks/race.md | 4 +- evalscope/docs/en/benchmarks/real_world_qa.md | 4 +- evalscope/docs/en/benchmarks/ref_adv_s.md | 155 + evalscope/docs/en/benchmarks/refcoco.md | 2 - .../docs/en/benchmarks/researchrubrics.md | 206 ++ evalscope/docs/en/benchmarks/sanskriti.md | 131 + evalscope/docs/en/benchmarks/scicode.md | 33 +- evalscope/docs/en/benchmarks/science_qa.md | 4 +- evalscope/docs/en/benchmarks/sciq.md | 4 +- .../docs/en/benchmarks/screenspot_pro.md | 151 + .../docs/en/benchmarks/seed_bench_2_plus.md | 4 +- evalscope/docs/en/benchmarks/seed_tts_eval.md | 2 - evalscope/docs/en/benchmarks/simple_qa.md | 2 - evalscope/docs/en/benchmarks/simple_vqa.md | 4 +- evalscope/docs/en/benchmarks/siqa.md | 4 +- evalscope/docs/en/benchmarks/skillsbench.md | 117 + evalscope/docs/en/benchmarks/slake.md | 161 + evalscope/docs/en/benchmarks/super_gpqa.md | 4 +- evalscope/docs/en/benchmarks/surds.md | 170 ++ .../docs/en/benchmarks/swe_bench_lite.md | 4 +- .../en/benchmarks/swe_bench_lite_agentic.md | 26 +- .../swe_bench_multilingual_agentic.md | 26 +- evalscope/docs/en/benchmarks/swe_bench_pro.md | 16 +- .../docs/en/benchmarks/swe_bench_verified.md | 4 +- .../benchmarks/swe_bench_verified_agentic.md | 26 +- .../en/benchmarks/swe_bench_verified_mini.md | 4 +- .../swe_bench_verified_mini_agentic.md | 26 +- evalscope/docs/en/benchmarks/tau2_bench.md | 6 +- evalscope/docs/en/benchmarks/tau3_bench.md | 6 +- evalscope/docs/en/benchmarks/tau_bench.md | 6 +- .../docs/en/benchmarks/terminal_bench_v2.md | 8 +- .../docs/en/benchmarks/terminal_bench_v2_1.md | 8 +- evalscope/docs/en/benchmarks/tifa160.md | 2 - evalscope/docs/en/benchmarks/tir_bench.md | 8 +- evalscope/docs/en/benchmarks/tool_bench.md | 4 +- evalscope/docs/en/benchmarks/toolathlon.md | 169 ++ evalscope/docs/en/benchmarks/torgo.md | 2 - evalscope/docs/en/benchmarks/trivia_qa.md | 16 +- .../docs/en/benchmarks/triviaqa_indic.md | 131 + evalscope/docs/en/benchmarks/truthful_qa.md | 11 +- evalscope/docs/en/benchmarks/tvbench.md | 154 + evalscope/docs/en/benchmarks/tweebank_ner.md | 4 +- evalscope/docs/en/benchmarks/tweet_ner_7.md | 4 +- evalscope/docs/en/benchmarks/videomme_v2.md | 9 +- evalscope/docs/en/benchmarks/visfactor.md | 135 + evalscope/docs/en/benchmarks/visulogic.md | 4 +- .../docs/en/benchmarks/vlms_are_biased.md | 147 + evalscope/docs/en/benchmarks/vqav2.md | 2 - evalscope/docs/en/benchmarks/vstar_bench.md | 4 +- evalscope/docs/en/benchmarks/vtcbench.md | 167 ++ evalscope/docs/en/benchmarks/wenet_speech.md | 2 - evalscope/docs/en/benchmarks/wide_search.md | 222 ++ evalscope/docs/en/benchmarks/winogrande.md | 4 +- evalscope/docs/en/benchmarks/wmt24pp.md | 2 - evalscope/docs/en/benchmarks/wnut2017.md | 4 +- evalscope/docs/en/benchmarks/world_vqa.md | 4 +- .../docs/en/benchmarks/zebralogicbench.md | 2 - evalscope/docs/en/benchmarks/zerobench.md | 4 +- .../docs/en/best_practice/index_collection.md | 13 +- evalscope/docs/en/best_practice/iquiz.md | 12 +- evalscope/docs/en/best_practice/qwen3.md | 16 +- .../docs/en/best_practice/qwen3_coder.md | 47 +- evalscope/docs/en/best_practice/qwen3_omni.md | 4 +- evalscope/docs/en/get_started/basic_usage.md | 33 +- evalscope/docs/en/get_started/faq.md | 45 +- evalscope/docs/en/get_started/parameters.md | 123 +- .../en/get_started/supported_dataset/agent.md | 28 +- .../en/get_started/supported_dataset/aigc.md | 2 +- .../en/get_started/supported_dataset/llm.md | 54 +- .../en/get_started/supported_dataset/vlm.md | 52 +- .../docs/en/get_started/visualization.md | 6 +- evalscope/docs/en/third_party/deepsearchqa.md | 134 + evalscope/docs/en/third_party/gaia.md | 2 +- evalscope/docs/en/third_party/index.md | 5 + evalscope/docs/en/third_party/miniwob.md | 62 + .../docs/en/third_party/needle_haystack.md | 10 +- evalscope/docs/en/third_party/skillsbench.md | 103 + evalscope/docs/en/third_party/swe_bench.md | 2 +- evalscope/docs/en/third_party/tau2_bench.md | 10 +- evalscope/docs/en/third_party/tau3_bench.md | 2 +- evalscope/docs/en/third_party/tau_bench.md | 8 +- .../docs/en/third_party/terminal_bench.md | 22 +- evalscope/docs/en/third_party/toolathlon.md | 139 + evalscope/docs/en/third_party/wide_search.md | 143 + evalscope/docs/en/user_guides/agent/bridge.md | 11 +- evalscope/docs/en/user_guides/agent/index.md | 2 +- evalscope/docs/en/user_guides/agent/native.md | 35 +- .../docs/en/user_guides/aigc/image_edit.md | 40 +- evalscope/docs/en/user_guides/aigc/t2i.md | 15 +- evalscope/docs/en/user_guides/arena.md | 95 +- evalscope/docs/en/user_guides/sandbox.md | 4 +- .../en/user_guides/stress_test/examples.md | 452 +-- .../en/user_guides/stress_test/multi_turn.md | 177 +- .../en/user_guides/stress_test/parameters.md | 213 +- .../en/user_guides/stress_test/quick_start.md | 14 +- .../docs/zh/advanced_guides/add_benchmark.md | 134 +- .../zh/advanced_guides/custom_dataset/clip.md | 2 +- .../custom_dataset/embedding.md | 2 +- .../zh/advanced_guides/custom_dataset/llm.md | 83 +- .../zh/advanced_guides/custom_dataset/vlm.md | 173 +- evalscope/docs/zh/benchmarks/a_okvqa.md | 4 +- evalscope/docs/zh/benchmarks/aa_lcr.md | 4 +- evalscope/docs/zh/benchmarks/acebench.md | 242 +- evalscope/docs/zh/benchmarks/agieval.md | 152 + evalscope/docs/zh/benchmarks/ai2d.md | 4 +- evalscope/docs/zh/benchmarks/aime24.md | 30 +- evalscope/docs/zh/benchmarks/aime25.md | 4 +- evalscope/docs/zh/benchmarks/aime26.md | 4 +- .../docs/zh/benchmarks/air_bench_chat.md | 55 +- .../zh/benchmarks/air_bench_foundation.md | 4 +- evalscope/docs/zh/benchmarks/alpaca_eval.md | 4 +- evalscope/docs/zh/benchmarks/amc.md | 4 +- evalscope/docs/zh/benchmarks/anat_em.md | 4 +- evalscope/docs/zh/benchmarks/arc.md | 4 +- evalscope/docs/zh/benchmarks/arc_agi_2.md | 115 + evalscope/docs/zh/benchmarks/arc_indic.md | 131 + evalscope/docs/zh/benchmarks/arena_hard.md | 4 +- evalscope/docs/zh/benchmarks/arxivmath.md | 30 +- .../docs/zh/benchmarks/arxivrollbench.md | 4 +- .../docs/zh/benchmarks/arxivrollbench_full.md | 4 +- .../docs/zh/benchmarks/automation_bench.md | 134 + evalscope/docs/zh/benchmarks/baby_vision.md | 52 +- evalscope/docs/zh/benchmarks/bbh.md | 20 +- evalscope/docs/zh/benchmarks/bc2gm.md | 4 +- evalscope/docs/zh/benchmarks/bc4chemd.md | 4 +- evalscope/docs/zh/benchmarks/bc5cdr.md | 4 +- evalscope/docs/zh/benchmarks/bfcl_v3.md | 4 +- evalscope/docs/zh/benchmarks/bfcl_v4.md | 4 +- .../zh/benchmarks/bhasha_bench_multi_ayur.md | 148 + .../benchmarks/bhasha_bench_multi_finance.md | 88 + .../benchmarks/bhasha_bench_multi_krishi.md | 148 + .../zh/benchmarks/bhasha_bench_multi_legal.md | 148 + .../docs/zh/benchmarks/bhashabenchv1_ayur.md | 128 + .../zh/benchmarks/bhashabenchv1_finance.md | 128 + .../zh/benchmarks/bhashabenchv1_krishi.md | 128 + .../docs/zh/benchmarks/bhashabenchv1_legal.md | 128 + evalscope/docs/zh/benchmarks/bigcodebench.md | 11 +- .../docs/zh/benchmarks/bigcodebench_hard.md | 19 +- evalscope/docs/zh/benchmarks/biomix_qa.md | 4 +- evalscope/docs/zh/benchmarks/blink.md | 4 +- .../zh/benchmarks/broad_twitter_corpus.md | 4 +- evalscope/docs/zh/benchmarks/browsecomp.md | 2 +- evalscope/docs/zh/benchmarks/cc_bench.md | 4 +- evalscope/docs/zh/benchmarks/cc_ocr_v2.md | 166 ++ evalscope/docs/zh/benchmarks/ceval.md | 4 +- evalscope/docs/zh/benchmarks/chartqa.md | 2 +- evalscope/docs/zh/benchmarks/charxiv.md | 46 +- .../docs/zh/benchmarks/chinese_simpleqa.md | 2 +- evalscope/docs/zh/benchmarks/cl_bench.md | 6 +- evalscope/docs/zh/benchmarks/claw_eval.md | 128 + evalscope/docs/zh/benchmarks/cmath.md | 26 +- evalscope/docs/zh/benchmarks/cmmlu.md | 50 +- evalscope/docs/zh/benchmarks/cmmmu.md | 20 +- evalscope/docs/zh/benchmarks/cmmu.md | 4 +- evalscope/docs/zh/benchmarks/coin_flip.md | 17 +- .../docs/zh/benchmarks/common_voice_15.md | 2 +- .../docs/zh/benchmarks/commonsense_qa.md | 4 +- .../docs/zh/benchmarks/competition_math.md | 30 +- evalscope/docs/zh/benchmarks/conll2003.md | 4 +- evalscope/docs/zh/benchmarks/conllpp.md | 4 +- evalscope/docs/zh/benchmarks/copious.md | 4 +- evalscope/docs/zh/benchmarks/count_qa.md | 92 + evalscope/docs/zh/benchmarks/cross_ner.md | 4 +- .../docs/zh/benchmarks/data_collection.md | 4 +- evalscope/docs/zh/benchmarks/deep_swe.md | 125 + evalscope/docs/zh/benchmarks/deepsearchqa.md | 100 + evalscope/docs/zh/benchmarks/docmath.md | 4 +- evalscope/docs/zh/benchmarks/docvqa.md | 2 +- evalscope/docs/zh/benchmarks/drivel_binary.md | 4 +- .../docs/zh/benchmarks/drivel_multilabel.md | 2 +- .../docs/zh/benchmarks/drivel_selection.md | 4 +- .../docs/zh/benchmarks/drivel_writing.md | 30 +- evalscope/docs/zh/benchmarks/drop.md | 32 +- .../docs/zh/benchmarks/emb_spatial_bench.md | 149 + evalscope/docs/zh/benchmarks/eq_bench.md | 2 +- evalscope/docs/zh/benchmarks/erqa.md | 4 +- evalscope/docs/zh/benchmarks/evalmuse.md | 2 +- evalscope/docs/zh/benchmarks/fin_ner.md | 4 +- evalscope/docs/zh/benchmarks/fleurs.md | 2 +- evalscope/docs/zh/benchmarks/frames.md | 4 +- evalscope/docs/zh/benchmarks/gaia.md | 36 +- evalscope/docs/zh/benchmarks/gdpval.md | 30 +- evalscope/docs/zh/benchmarks/gedit.md | 20 +- evalscope/docs/zh/benchmarks/genai_bench.md | 30 +- evalscope/docs/zh/benchmarks/general_arena.md | 4 +- evalscope/docs/zh/benchmarks/general_fc.md | 2 +- evalscope/docs/zh/benchmarks/general_mcq.md | 4 +- evalscope/docs/zh/benchmarks/general_qa.md | 2 +- evalscope/docs/zh/benchmarks/general_t2i.md | 2 +- evalscope/docs/zh/benchmarks/general_vmcq.md | 24 +- evalscope/docs/zh/benchmarks/general_vqa.md | 21 +- evalscope/docs/zh/benchmarks/genia_ner.md | 4 +- evalscope/docs/zh/benchmarks/gpqa_diamond.md | 4 +- evalscope/docs/zh/benchmarks/gsm8k.md | 28 +- evalscope/docs/zh/benchmarks/gsm8k_indic.md | 130 + evalscope/docs/zh/benchmarks/gsm8k_v.md | 4 +- .../docs/zh/benchmarks/hallusion_bench.md | 26 +- evalscope/docs/zh/benchmarks/halueval.md | 4 +- evalscope/docs/zh/benchmarks/harvey_ner.md | 4 +- evalscope/docs/zh/benchmarks/health_bench.md | 2 +- evalscope/docs/zh/benchmarks/hellaswag.md | 4 +- evalscope/docs/zh/benchmarks/hellaswag_hi.md | 112 + evalscope/docs/zh/benchmarks/hipho.md | 171 ++ evalscope/docs/zh/benchmarks/hle.md | 4 +- evalscope/docs/zh/benchmarks/hmmt25.md | 30 +- evalscope/docs/zh/benchmarks/hmmt26.md | 30 +- evalscope/docs/zh/benchmarks/hmmt_nov25.md | 112 + evalscope/docs/zh/benchmarks/hpdv2.md | 2 +- evalscope/docs/zh/benchmarks/humaneval.md | 4 +- .../docs/zh/benchmarks/humaneval_plus.md | 4 +- evalscope/docs/zh/benchmarks/ifbench.md | 26 +- evalscope/docs/zh/benchmarks/ifeval.md | 26 +- .../docs/zh/benchmarks/imo_answerbench.md | 32 +- evalscope/docs/zh/benchmarks/indic_boolq.md | 132 + evalscope/docs/zh/benchmarks/indic_param.md | 139 + evalscope/docs/zh/benchmarks/infovqa.md | 2 +- evalscope/docs/zh/benchmarks/iquiz.md | 4 +- evalscope/docs/zh/benchmarks/jnlpba.md | 4 +- evalscope/docs/zh/benchmarks/jnlpba_rare.md | 4 +- evalscope/docs/zh/benchmarks/job_bench.md | 153 + evalscope/docs/zh/benchmarks/k2_verifier.md | 2 +- evalscope/docs/zh/benchmarks/kimi_verifier.md | 18 +- evalscope/docs/zh/benchmarks/kina.md | 126 + evalscope/docs/zh/benchmarks/librispeech.md | 2 +- .../docs/zh/benchmarks/live_code_bench.md | 24 +- evalscope/docs/zh/benchmarks/locomo.md | 2 +- evalscope/docs/zh/benchmarks/logi_qa.md | 4 +- evalscope/docs/zh/benchmarks/longbench_v2.md | 4 +- evalscope/docs/zh/benchmarks/longmemeval.md | 4 +- .../docs/zh/benchmarks/maritime_bench.md | 4 +- .../docs/zh/benchmarks/maritime_ocr_bench.md | 4 +- evalscope/docs/zh/benchmarks/math_500.md | 26 +- evalscope/docs/zh/benchmarks/math_qa.md | 4 +- evalscope/docs/zh/benchmarks/math_verse.md | 4 +- evalscope/docs/zh/benchmarks/math_vision.md | 4 +- evalscope/docs/zh/benchmarks/math_vista.md | 34 +- evalscope/docs/zh/benchmarks/mbpp.md | 26 +- evalscope/docs/zh/benchmarks/mbpp_plus.md | 4 +- evalscope/docs/zh/benchmarks/mcp_atlas.md | 26 +- evalscope/docs/zh/benchmarks/measure_bench.md | 142 + evalscope/docs/zh/benchmarks/med_mcqa.md | 4 +- evalscope/docs/zh/benchmarks/medxpertqa.md | 167 ++ evalscope/docs/zh/benchmarks/mgsm.md | 32 +- evalscope/docs/zh/benchmarks/mia_bench.md | 30 +- evalscope/docs/zh/benchmarks/micro_vqa.md | 4 +- evalscope/docs/zh/benchmarks/milu.md | 137 + evalscope/docs/zh/benchmarks/minerva_math.md | 32 +- .../docs/zh/benchmarks/minimax_verifier.md | 2 +- evalscope/docs/zh/benchmarks/miniwob.md | 133 + .../docs/zh/benchmarks/mit_movie_trivia.md | 4 +- .../docs/zh/benchmarks/mit_restaurant.md | 4 +- evalscope/docs/zh/benchmarks/mm_bench.md | 4 +- evalscope/docs/zh/benchmarks/mm_star.md | 30 +- evalscope/docs/zh/benchmarks/mmau.md | 4 +- evalscope/docs/zh/benchmarks/mmlu.md | 4 +- evalscope/docs/zh/benchmarks/mmlu_pro.md | 4 +- evalscope/docs/zh/benchmarks/mmlu_redux.md | 4 +- evalscope/docs/zh/benchmarks/mmmlu.md | 4 +- evalscope/docs/zh/benchmarks/mmmu.md | 4 +- evalscope/docs/zh/benchmarks/mmmu_pro.md | 4 +- evalscope/docs/zh/benchmarks/mri_mcqa.md | 4 +- evalscope/docs/zh/benchmarks/msr_vtt.md | 19 +- evalscope/docs/zh/benchmarks/msvd.md | 17 +- evalscope/docs/zh/benchmarks/multi_if.md | 19 +- evalscope/docs/zh/benchmarks/multi_nerd.md | 4 +- .../docs/zh/benchmarks/multiple_humaneval.md | 4 +- evalscope/docs/zh/benchmarks/multiple_mbpp.md | 4 +- evalscope/docs/zh/benchmarks/music_trivia.md | 4 +- evalscope/docs/zh/benchmarks/musr.md | 4 +- evalscope/docs/zh/benchmarks/mvbench.md | 25 +- evalscope/docs/zh/benchmarks/ncbi.md | 4 +- .../docs/zh/benchmarks/needle_haystack.md | 4 +- evalscope/docs/zh/benchmarks/ocr_bench.md | 32 +- evalscope/docs/zh/benchmarks/ocr_bench_v2.md | 4 +- evalscope/docs/zh/benchmarks/officeqa.md | 114 + evalscope/docs/zh/benchmarks/olmocr_bench.md | 180 ++ .../docs/zh/benchmarks/olympiad_bench.md | 4 +- evalscope/docs/zh/benchmarks/omni_bench.md | 4 +- .../docs/zh/benchmarks/omni_doc_bench.md | 509 +--- .../docs/zh/benchmarks/omni_doc_bench_v1_6.md | 178 ++ .../docs/zh/benchmarks/one_million_bench.md | 150 + evalscope/docs/zh/benchmarks/ontonotes5.md | 4 +- evalscope/docs/zh/benchmarks/openai_mrcr.md | 2 +- .../docs/zh/benchmarks/perception_bench.md | 148 + .../perspective_gap_prompt_writing.md | 164 ++ .../perspective_gap_role_assignment.md | 165 ++ evalscope/docs/zh/benchmarks/phyx_mc.md | 144 + evalscope/docs/zh/benchmarks/phyx_oe.md | 146 + evalscope/docs/zh/benchmarks/piqa.md | 4 +- evalscope/docs/zh/benchmarks/plawbench.md | 142 + evalscope/docs/zh/benchmarks/pmc_vqa.md | 135 + evalscope/docs/zh/benchmarks/poly_math.md | 4 +- evalscope/docs/zh/benchmarks/pope.md | 23 +- evalscope/docs/zh/benchmarks/prbench.md | 229 ++ evalscope/docs/zh/benchmarks/process_bench.md | 2 +- evalscope/docs/zh/benchmarks/pubmedqa.md | 4 +- evalscope/docs/zh/benchmarks/qasc.md | 4 +- evalscope/docs/zh/benchmarks/race.md | 4 +- evalscope/docs/zh/benchmarks/real_world_qa.md | 4 +- evalscope/docs/zh/benchmarks/ref_adv_s.md | 155 + evalscope/docs/zh/benchmarks/refcoco.md | 2 +- .../docs/zh/benchmarks/researchrubrics.md | 192 ++ evalscope/docs/zh/benchmarks/sanskriti.md | 124 + evalscope/docs/zh/benchmarks/scicode.md | 2 +- evalscope/docs/zh/benchmarks/science_qa.md | 4 +- evalscope/docs/zh/benchmarks/sciq.md | 22 +- .../docs/zh/benchmarks/screenspot_pro.md | 156 + .../docs/zh/benchmarks/seed_bench_2_plus.md | 4 +- evalscope/docs/zh/benchmarks/seed_tts_eval.md | 2 +- evalscope/docs/zh/benchmarks/simple_qa.md | 2 +- evalscope/docs/zh/benchmarks/simple_vqa.md | 4 +- evalscope/docs/zh/benchmarks/siqa.md | 4 +- evalscope/docs/zh/benchmarks/skillsbench.md | 113 + evalscope/docs/zh/benchmarks/slake.md | 147 + evalscope/docs/zh/benchmarks/super_gpqa.md | 4 +- evalscope/docs/zh/benchmarks/surds.md | 163 ++ .../docs/zh/benchmarks/swe_bench_lite.md | 4 +- .../zh/benchmarks/swe_bench_lite_agentic.md | 45 +- .../swe_bench_multilingual_agentic.md | 32 +- evalscope/docs/zh/benchmarks/swe_bench_pro.md | 40 +- .../docs/zh/benchmarks/swe_bench_verified.md | 4 +- .../benchmarks/swe_bench_verified_agentic.md | 38 +- .../zh/benchmarks/swe_bench_verified_mini.md | 4 +- .../swe_bench_verified_mini_agentic.md | 32 +- evalscope/docs/zh/benchmarks/tau2_bench.md | 42 +- evalscope/docs/zh/benchmarks/tau3_bench.md | 22 +- evalscope/docs/zh/benchmarks/tau_bench.md | 27 +- .../docs/zh/benchmarks/terminal_bench_v2.md | 46 +- .../docs/zh/benchmarks/terminal_bench_v2_1.md | 45 +- evalscope/docs/zh/benchmarks/tifa160.md | 28 +- evalscope/docs/zh/benchmarks/tir_bench.md | 16 +- evalscope/docs/zh/benchmarks/tool_bench.md | 4 +- evalscope/docs/zh/benchmarks/toolathlon.md | 163 ++ evalscope/docs/zh/benchmarks/torgo.md | 32 +- evalscope/docs/zh/benchmarks/trivia_qa.md | 22 +- .../docs/zh/benchmarks/triviaqa_indic.md | 128 + evalscope/docs/zh/benchmarks/truthful_qa.md | 2 +- evalscope/docs/zh/benchmarks/tvbench.md | 153 + evalscope/docs/zh/benchmarks/tweebank_ner.md | 4 +- evalscope/docs/zh/benchmarks/tweet_ner_7.md | 4 +- evalscope/docs/zh/benchmarks/videomme_v2.md | 21 +- evalscope/docs/zh/benchmarks/visfactor.md | 134 + evalscope/docs/zh/benchmarks/visulogic.md | 4 +- .../docs/zh/benchmarks/vlms_are_biased.md | 147 + evalscope/docs/zh/benchmarks/vqav2.md | 2 +- evalscope/docs/zh/benchmarks/vstar_bench.md | 4 +- evalscope/docs/zh/benchmarks/vtcbench.md | 165 ++ evalscope/docs/zh/benchmarks/wenet_speech.md | 2 +- evalscope/docs/zh/benchmarks/wide_search.md | 218 ++ evalscope/docs/zh/benchmarks/winogrande.md | 4 +- evalscope/docs/zh/benchmarks/wmt24pp.md | 2 +- evalscope/docs/zh/benchmarks/wnut2017.md | 4 +- evalscope/docs/zh/benchmarks/world_vqa.md | 4 +- .../docs/zh/benchmarks/zebralogicbench.md | 2 +- evalscope/docs/zh/benchmarks/zerobench.md | 32 +- evalscope/docs/zh/best_practice/general_fc.md | 2 +- .../docs/zh/best_practice/index_collection.md | 6 +- evalscope/docs/zh/best_practice/iquiz.md | 12 +- evalscope/docs/zh/best_practice/qwen3.md | 16 +- .../docs/zh/best_practice/qwen3_coder.md | 47 +- evalscope/docs/zh/best_practice/qwen3_omni.md | 4 +- .../zh/best_practice/swift_integration.md | 2 +- evalscope/docs/zh/blog/RAG/multimodal_RAG.md | 2 +- evalscope/docs/zh/get_started/basic_usage.md | 33 +- evalscope/docs/zh/get_started/faq.md | 46 +- evalscope/docs/zh/get_started/parameters.md | 126 +- .../zh/get_started/supported_dataset/agent.md | 28 +- .../zh/get_started/supported_dataset/aigc.md | 2 +- .../zh/get_started/supported_dataset/llm.md | 54 +- .../zh/get_started/supported_dataset/vlm.md | 52 +- .../docs/zh/get_started/visualization.md | 6 +- evalscope/docs/zh/third_party/deepsearchqa.md | 129 + evalscope/docs/zh/third_party/gaia.md | 2 +- evalscope/docs/zh/third_party/index.md | 7 +- evalscope/docs/zh/third_party/miniwob.md | 59 + .../docs/zh/third_party/needle_haystack.md | 10 +- evalscope/docs/zh/third_party/skillsbench.md | 103 + evalscope/docs/zh/third_party/swe_bench.md | 2 +- evalscope/docs/zh/third_party/tau2_bench.md | 8 +- evalscope/docs/zh/third_party/tau3_bench.md | 2 +- evalscope/docs/zh/third_party/tau_bench.md | 7 +- .../docs/zh/third_party/terminal_bench.md | 18 +- evalscope/docs/zh/third_party/toolathlon.md | 139 + evalscope/docs/zh/third_party/wide_search.md | 139 + evalscope/docs/zh/user_guides/agent/bridge.md | 12 +- evalscope/docs/zh/user_guides/agent/index.md | 2 +- evalscope/docs/zh/user_guides/agent/native.md | 34 +- .../docs/zh/user_guides/aigc/image_edit.md | 40 +- evalscope/docs/zh/user_guides/aigc/t2i.md | 15 +- evalscope/docs/zh/user_guides/arena.md | 94 +- .../backend/rageval_backend/index.md | 2 +- .../user_guides/backend/vlmevalkit_backend.md | 6 +- evalscope/docs/zh/user_guides/sandbox.md | 4 +- .../zh/user_guides/stress_test/examples.md | 524 ++-- .../zh/user_guides/stress_test/multi_turn.md | 171 +- .../zh/user_guides/stress_test/parameters.md | 212 +- .../zh/user_guides/stress_test/quick_start.md | 16 +- evalscope/evalscope/__init__.py | 5 +- evalscope/evalscope/agent/__init__.py | 8 +- .../evalscope/agent/environments/__init__.py | 6 +- .../evalscope/agent/environments/enclave.py | 7 +- .../evalscope/agent/environments/local.py | 2 +- .../evalscope/agent/external/__init__.py | 1 + evalscope/evalscope/agent/external/adapter.py | 3 +- .../agent/external/bridge/_sse_common.py | 2 +- .../evalscope/agent/external/bridge/server.py | 353 ++- .../agent/external/bridge/sse_anthropic.py | 85 +- .../agent/external/bridge/sse_gemini.py | 1 + .../agent/external/bridge/sse_openai.py | 47 +- .../agent/external/bridge/sse_responses.py | 82 +- .../agent/external/bridge/trace_recorder.py | 28 +- .../external/bridge/translate_anthropic.py | 27 +- .../agent/external/bridge/translate_gemini.py | 49 +- .../agent/external/bridge/translate_openai.py | 34 +- .../external/bridge/translate_responses.py | 106 +- evalscope/evalscope/agent/external/config.py | 4 +- .../agent/external/runners/__init__.py | 1 + .../evalscope/agent/external/runners/base.py | 3 +- .../agent/external/runners/claude_code.py | 8 +- .../evalscope/agent/external/runners/codex.py | 9 +- .../agent/external/runners/gemini_cli.py | 9 +- .../agent/external/runners/hermes.py | 19 +- .../agent/external/runners/install_helper.py | 1 + .../evalscope/agent/external/runners/mock.py | 1 + .../agent/external/runners/opencode.py | 22 +- evalscope/evalscope/agent/runner.py | 2 +- evalscope/evalscope/agent/skills.py | 11 +- .../agent/strategies/function_calling.py | 42 +- evalscope/evalscope/agent/strategies/react.py | 9 +- .../strategies/swe_bench/_observation.py | 10 +- .../swe_bench/swe_bench_backticks.py | 34 +- .../swe_bench/swe_bench_toolcall.py | 24 +- evalscope/evalscope/agent/tools/bash.py | 14 +- .../evalscope/agent/tools/python_exec.py | 3 +- evalscope/evalscope/api/agent/__init__.py | 4 + evalscope/evalscope/api/agent/constants.py | 10 +- evalscope/evalscope/api/agent/loop.py | 102 +- evalscope/evalscope/api/agent/mcp/client.py | 1 + evalscope/evalscope/api/agent/mcp/source.py | 1 + evalscope/evalscope/api/agent/mcp/types.py | 3 +- evalscope/evalscope/api/agent/runner.py | 3 +- evalscope/evalscope/api/agent/strategy.py | 50 +- .../evalscope/api/agent/tool_executor.py | 19 +- evalscope/evalscope/api/agent/trace.py | 6 +- evalscope/evalscope/api/agent/types.py | 92 +- evalscope/evalscope/api/benchmark/__init__.py | 1 + .../api/benchmark/adapters/__init__.py | 1 + .../api/benchmark/adapters/agent_adapter.py | 14 +- .../adapters/audio_language_adapter.py | 1 + .../benchmark/adapters/browsergym_adapter.py | 523 ++++ .../adapters/default_data_adapter.py | 96 +- .../adapters/function_call_adapter.py | 2 + .../benchmark/adapters/image_edit_adapter.py | 1 + .../adapters/multi_choice_adapter.py | 5 +- .../benchmark/adapters/multi_turn_adapter.py | 7 +- .../api/benchmark/adapters/ner_adapter.py | 36 +- .../benchmark/adapters/text2image_adapter.py | 143 +- .../adapters/vision_language_adapter.py | 325 ++- .../evalscope/api/benchmark/benchmark.py | 31 +- evalscope/evalscope/api/benchmark/meta.py | 149 +- .../evalscope/api/benchmark/statistics.py | 12 +- evalscope/evalscope/api/dataset/dataset.py | 67 +- evalscope/evalscope/api/dataset/hub.py | 16 +- evalscope/evalscope/api/dataset/loader.py | 123 +- evalscope/evalscope/api/dataset/utils.py | 20 +- evalscope/evalscope/api/evaluator/cache.py | 110 +- evalscope/evalscope/api/evaluator/state.py | 12 +- evalscope/evalscope/api/judge/__init__.py | 48 + evalscope/evalscope/api/judge/aggregation.py | 251 ++ evalscope/evalscope/api/judge/contracts.py | 140 + evalscope/evalscope/api/judge/definition.py | 224 ++ evalscope/evalscope/api/judge/executor.py | 348 +++ evalscope/evalscope/api/judge/summary.py | 94 + evalscope/evalscope/api/judge/types.py | 210 ++ evalscope/evalscope/api/messages/__init__.py | 1 + .../evalscope/api/messages/chat_message.py | 13 +- evalscope/evalscope/api/messages/content.py | 3 +- .../evalscope/api/messages/perf_metrics.py | 4 +- .../evalscope/api/messages/request_stats.py | 125 + evalscope/evalscope/api/messages/utils.py | 2 +- evalscope/evalscope/api/metric/__init__.py | 16 +- evalscope/evalscope/api/metric/metric.py | 3 + evalscope/evalscope/api/metric/scorer.py | 110 +- evalscope/evalscope/api/metric/semantics.py | 321 ++ .../api/mixin/code_execution_sandbox_mixin.py | 14 +- .../evalscope/api/mixin/llm_judge_mixin.py | 372 ++- .../evalscope/api/model/generate_config.py | 17 +- evalscope/evalscope/api/model/model.py | 48 +- evalscope/evalscope/api/model/model_output.py | 5 +- evalscope/evalscope/api/registry.py | 10 +- .../evalscope/api/sandbox/config_builder.py | 1 + .../evalscope/api/sandbox/docker_image.py | 5 +- evalscope/evalscope/api/sandbox/engine.py | 9 +- evalscope/evalscope/api/sandbox/service.py | 367 ++- evalscope/evalscope/api/tool/tool_call.py | 3 +- evalscope/evalscope/api/tool/tool_info.py | 11 +- evalscope/evalscope/api/tool/utils.py | 8 +- evalscope/evalscope/arguments.py | 19 +- evalscope/evalscope/backend/base.py | 1 - .../backend/opencompass/api_meta_template.py | 9 +- .../backend/opencompass/backend_manager.py | 33 +- .../backend/opencompass/tasks/eval_api.py | 1 + .../backend/rag_eval/backend_manager.py | 79 +- .../rag_eval/clip_benchmark/arguments.py | 3 +- .../clip_benchmark/dataset_builder.py | 10 +- .../rag_eval/clip_benchmark/task_template.py | 9 +- .../clip_benchmark/tasks/image_caption.py | 1 + .../tasks/zeroshot_classification.py | 3 +- .../tasks/zeroshot_retrieval.py | 3 +- .../utils/webdataset_convert.py | 29 +- .../backend/rag_eval/models/__init__.py | 2 + .../evalscope/backend/rag_eval/models/base.py | 8 +- .../backend/rag_eval/models/encoder.py | 23 +- .../backend/rag_eval/models/reranker.py | 12 +- .../backend/rag_eval/models/utils.py | 3 + .../backend/rag_eval/mteb/arguments.py | 4 +- .../backend/rag_eval/mteb/custom_task.py | 14 +- .../backend/rag_eval/mteb/data_loader.py | 29 +- .../backend/rag_eval/mteb/task_template.py | 48 +- .../backend/rag_eval/ragas/arguments.py | 3 +- .../rag_eval/ragas/prompts/persona_prompt.py | 21 +- .../backend/rag_eval/ragas/task_template.py | 4 + .../ragas/tasks/build_distribution.py | 1 + .../rag_eval/ragas/tasks/build_transform.py | 3 +- .../ragas/tasks/testset_generation.py | 21 +- .../rag_eval/ragas/tasks/translate_prompt.py | 3 +- .../evalscope/backend/rag_eval/utils/clip.py | 54 +- .../evalscope/backend/rag_eval/utils/llm.py | 3 +- .../evalscope/backend/rag_eval/utils/tools.py | 1 + .../backend/vlm_eval_kit/backend_manager.py | 34 +- .../evalscope/benchmarks/_meta/a_okvqa.json | 10 +- .../evalscope/benchmarks/_meta/aa_lcr.json | 10 +- .../evalscope/benchmarks/_meta/acebench.json | 429 +-- .../evalscope/benchmarks/_meta/agieval.json | 10 +- .../evalscope/benchmarks/_meta/ai2d.json | 10 +- .../evalscope/benchmarks/_meta/aime24.json | 10 +- .../evalscope/benchmarks/_meta/aime25.json | 10 +- .../evalscope/benchmarks/_meta/aime26.json | 10 +- .../benchmarks/_meta/air_bench_chat.json | 21 +- .../_meta/air_bench_foundation.json | 10 +- .../benchmarks/_meta/alpaca_eval.json | 10 +- evalscope/evalscope/benchmarks/_meta/amc.json | 10 +- .../evalscope/benchmarks/_meta/anat_em.json | 15 +- evalscope/evalscope/benchmarks/_meta/arc.json | 10 +- .../evalscope/benchmarks/_meta/arc_agi_2.json | 17 +- .../evalscope/benchmarks/_meta/arc_indic.json | 184 ++ .../benchmarks/_meta/arena_hard.json | 10 +- .../evalscope/benchmarks/_meta/arxivmath.json | 10 +- .../benchmarks/_meta/arxivrollbench.json | 10 +- .../benchmarks/_meta/arxivrollbench_full.json | 10 +- .../benchmarks/_meta/automation_bench.json | 157 + .../benchmarks/_meta/baby_vision.json | 238 +- evalscope/evalscope/benchmarks/_meta/bbh.json | 12 +- .../evalscope/benchmarks/_meta/bc2gm.json | 15 +- .../evalscope/benchmarks/_meta/bc4chemd.json | 15 +- .../evalscope/benchmarks/_meta/bc5cdr.json | 15 +- .../evalscope/benchmarks/_meta/bfcl_v3.json | 10 +- .../evalscope/benchmarks/_meta/bfcl_v4.json | 10 +- .../_meta/bhasha_bench_multi_ayur.json | 294 ++ .../_meta/bhasha_bench_multi_finance.json | 71 + .../_meta/bhasha_bench_multi_krishi.json | 294 ++ .../_meta/bhasha_bench_multi_legal.json | 294 ++ .../benchmarks/_meta/bhashabenchv1_ayur.json | 94 + .../_meta/bhashabenchv1_finance.json | 94 + .../_meta/bhashabenchv1_krishi.json | 94 + .../benchmarks/_meta/bhashabenchv1_legal.json | 94 + .../benchmarks/_meta/bigcodebench.json | 17 +- .../benchmarks/_meta/bigcodebench_hard.json | 17 +- .../evalscope/benchmarks/_meta/biomix_qa.json | 10 +- .../evalscope/benchmarks/_meta/blink.json | 10 +- .../_meta/broad_twitter_corpus.json | 15 +- .../benchmarks/_meta/browsecomp.json | 7 +- .../evalscope/benchmarks/_meta/cc_bench.json | 10 +- .../evalscope/benchmarks/_meta/cc_ocr_v2.json | 777 +++++ .../evalscope/benchmarks/_meta/ceval.json | 10 +- .../evalscope/benchmarks/_meta/charxiv.json | 147 +- .../benchmarks/_meta/chinese_simpleqa.json | 7 +- .../evalscope/benchmarks/_meta/cl_bench.json | 10 +- .../evalscope/benchmarks/_meta/claw_eval.json | 15 +- .../evalscope/benchmarks/_meta/cmath.json | 10 +- .../evalscope/benchmarks/_meta/cmmlu.json | 22 +- .../evalscope/benchmarks/_meta/cmmmu.json | 10 +- .../evalscope/benchmarks/_meta/cmmu.json | 10 +- .../evalscope/benchmarks/_meta/coin_flip.json | 19 +- .../benchmarks/_meta/commonsense_qa.json | 10 +- .../benchmarks/_meta/competition_math.json | 10 +- .../evalscope/benchmarks/_meta/conll2003.json | 15 +- .../evalscope/benchmarks/_meta/conllpp.json | 15 +- .../evalscope/benchmarks/_meta/copious.json | 15 +- .../evalscope/benchmarks/_meta/count_qa.json | 163 ++ .../evalscope/benchmarks/_meta/cross_ner.json | 15 +- .../benchmarks/_meta/data_collection.json | 10 +- .../evalscope/benchmarks/_meta/deep_swe.json | 10 +- .../benchmarks/_meta/deepsearchqa.json | 72 + .../evalscope/benchmarks/_meta/docmath.json | 10 +- .../benchmarks/_meta/drivel_binary.json | 15 +- .../benchmarks/_meta/drivel_multilabel.json | 7 +- .../benchmarks/_meta/drivel_selection.json | 10 +- .../benchmarks/_meta/drivel_writing.json | 15 +- .../evalscope/benchmarks/_meta/drop.json | 15 +- .../benchmarks/_meta/emb_spatial_bench.json | 12 +- .../evalscope/benchmarks/_meta/erqa.json | 10 +- .../evalscope/benchmarks/_meta/evalmuse.json | 11 +- .../evalscope/benchmarks/_meta/fin_ner.json | 15 +- .../evalscope/benchmarks/_meta/frames.json | 10 +- .../evalscope/benchmarks/_meta/gaia.json | 14 +- .../evalscope/benchmarks/_meta/gedit.json | 20 +- .../benchmarks/_meta/genai_bench.json | 11 +- .../benchmarks/_meta/general_arena.json | 10 +- .../benchmarks/_meta/general_fc.json | 7 +- .../benchmarks/_meta/general_mcq.json | 10 +- .../benchmarks/_meta/general_qa.json | 10 +- .../benchmarks/_meta/general_vmcq.json | 14 +- .../benchmarks/_meta/general_vqa.json | 24 +- .../evalscope/benchmarks/_meta/genia_ner.json | 15 +- .../benchmarks/_meta/gpqa_diamond.json | 10 +- .../evalscope/benchmarks/_meta/gsm8k.json | 10 +- .../benchmarks/_meta/gsm8k_indic.json | 281 ++ .../evalscope/benchmarks/_meta/gsm8k_v.json | 10 +- .../benchmarks/_meta/hallusion_bench.json | 16 +- .../evalscope/benchmarks/_meta/halueval.json | 15 +- .../benchmarks/_meta/harvey_ner.json | 15 +- .../benchmarks/_meta/health_bench.json | 7 +- .../evalscope/benchmarks/_meta/hellaswag.json | 10 +- .../benchmarks/_meta/hellaswag_hi.json | 82 + .../evalscope/benchmarks/_meta/hipho.json | 678 +++++ evalscope/evalscope/benchmarks/_meta/hle.json | 10 +- .../evalscope/benchmarks/_meta/hmmt25.json | 10 +- .../evalscope/benchmarks/_meta/hmmt26.json | 10 +- .../benchmarks/_meta/hmmt_nov25.json | 81 + .../evalscope/benchmarks/_meta/hpdv2.json | 11 +- .../evalscope/benchmarks/_meta/humaneval.json | 17 +- .../benchmarks/_meta/humaneval_plus.json | 17 +- .../evalscope/benchmarks/_meta/ifbench.json | 21 +- .../evalscope/benchmarks/_meta/ifeval.json | 21 +- .../benchmarks/_meta/imo_answerbench.json | 10 +- .../benchmarks/_meta/indic_boolq.json | 182 ++ .../benchmarks/_meta/indic_param.json | 195 ++ .../evalscope/benchmarks/_meta/iquiz.json | 10 +- .../evalscope/benchmarks/_meta/jnlpba.json | 15 +- .../benchmarks/_meta/jnlpba_rare.json | 15 +- .../evalscope/benchmarks/_meta/job_bench.json | 134 + .../benchmarks/_meta/k2_verifier.json | 7 +- .../benchmarks/_meta/kimi_verifier.json | 21 +- .../evalscope/benchmarks/_meta/kina.json | 12 +- .../benchmarks/_meta/live_code_bench.json | 23 +- .../evalscope/benchmarks/_meta/locomo.json | 9 +- .../evalscope/benchmarks/_meta/logi_qa.json | 10 +- .../benchmarks/_meta/longbench_v2.json | 10 +- .../benchmarks/_meta/longmemeval.json | 17 +- .../benchmarks/_meta/maritime_bench.json | 10 +- .../benchmarks/_meta/maritime_ocr_bench.json | 10 +- .../evalscope/benchmarks/_meta/math_500.json | 10 +- .../evalscope/benchmarks/_meta/math_qa.json | 10 +- .../benchmarks/_meta/math_verse.json | 10 +- .../benchmarks/_meta/math_vision.json | 10 +- .../benchmarks/_meta/math_vista.json | 10 +- .../evalscope/benchmarks/_meta/mbpp.json | 17 +- .../evalscope/benchmarks/_meta/mbpp_plus.json | 17 +- .../evalscope/benchmarks/_meta/mcp_atlas.json | 21 +- .../benchmarks/_meta/measure_bench.json | 17 +- .../evalscope/benchmarks/_meta/med_mcqa.json | 10 +- .../benchmarks/_meta/medxpertqa.json | 178 ++ .../evalscope/benchmarks/_meta/mgsm.json | 10 +- .../evalscope/benchmarks/_meta/mia_bench.json | 18 +- .../evalscope/benchmarks/_meta/micro_vqa.json | 10 +- .../evalscope/benchmarks/_meta/milu.json | 183 ++ .../benchmarks/_meta/minerva_math.json | 10 +- .../benchmarks/_meta/minimax_verifier.json | 7 +- .../evalscope/benchmarks/_meta/miniwob.json | 113 + .../benchmarks/_meta/mit_movie_trivia.json | 15 +- .../benchmarks/_meta/mit_restaurant.json | 15 +- .../evalscope/benchmarks/_meta/mm_bench.json | 10 +- .../evalscope/benchmarks/_meta/mm_star.json | 10 +- .../evalscope/benchmarks/_meta/mmau.json | 10 +- .../evalscope/benchmarks/_meta/mmlu.json | 10 +- .../evalscope/benchmarks/_meta/mmlu_pro.json | 10 +- .../benchmarks/_meta/mmlu_redux.json | 10 +- .../evalscope/benchmarks/_meta/mmmlu.json | 10 +- .../evalscope/benchmarks/_meta/mmmu.json | 10 +- .../evalscope/benchmarks/_meta/mmmu_pro.json | 10 +- .../evalscope/benchmarks/_meta/mri_mcqa.json | 10 +- .../evalscope/benchmarks/_meta/msr_vtt.json | 7 +- .../evalscope/benchmarks/_meta/msvd.json | 7 +- .../evalscope/benchmarks/_meta/multi_if.json | 21 +- .../benchmarks/_meta/multi_nerd.json | 15 +- .../benchmarks/_meta/multiple_humaneval.json | 17 +- .../benchmarks/_meta/multiple_mbpp.json | 17 +- .../benchmarks/_meta/music_trivia.json | 10 +- .../evalscope/benchmarks/_meta/musr.json | 10 +- .../evalscope/benchmarks/_meta/mvbench.json | 10 +- .../evalscope/benchmarks/_meta/ncbi.json | 15 +- .../benchmarks/_meta/needle_haystack.json | 10 +- .../evalscope/benchmarks/_meta/ocr_bench.json | 10 +- .../benchmarks/_meta/ocr_bench_v2.json | 10 +- .../evalscope/benchmarks/_meta/officeqa.json | 14 +- .../benchmarks/_meta/olmocr_bench.json | 350 +++ .../benchmarks/_meta/olympiad_bench.json | 10 +- .../benchmarks/_meta/omni_bench.json | 10 +- .../benchmarks/_meta/omni_doc_bench.json | 492 +--- .../benchmarks/_meta/omni_doc_bench_v1_6.json | 176 ++ .../benchmarks/_meta/one_million_bench.json | 190 ++ .../benchmarks/_meta/ontonotes5.json | 15 +- .../benchmarks/_meta/openai_mrcr.json | 9 +- .../benchmarks/_meta/perception_bench.json | 547 ++++ .../_meta/perspective_gap_prompt_writing.json | 7 +- .../perspective_gap_role_assignment.json | 7 +- .../evalscope/benchmarks/_meta/phyx_mc.json | 368 +++ .../evalscope/benchmarks/_meta/phyx_oe.json | 368 +++ .../evalscope/benchmarks/_meta/piqa.json | 10 +- .../evalscope/benchmarks/_meta/plawbench.json | 124 + .../evalscope/benchmarks/_meta/pmc_vqa.json | 154 + .../evalscope/benchmarks/_meta/poly_math.json | 10 +- .../evalscope/benchmarks/_meta/pope.json | 17 +- .../evalscope/benchmarks/_meta/prbench.json | 219 ++ .../benchmarks/_meta/process_bench.json | 7 +- .../evalscope/benchmarks/_meta/pubmedqa.json | 15 +- .../evalscope/benchmarks/_meta/qasc.json | 10 +- .../evalscope/benchmarks/_meta/race.json | 10 +- .../benchmarks/_meta/real_world_qa.json | 10 +- .../evalscope/benchmarks/_meta/ref_adv_s.json | 197 ++ .../evalscope/benchmarks/_meta/refcoco.json | 7 +- .../benchmarks/_meta/researchrubrics.json | 21 +- .../evalscope/benchmarks/_meta/sanskriti.json | 114 + .../evalscope/benchmarks/_meta/scicode.json | 15 +- .../benchmarks/_meta/science_qa.json | 10 +- .../evalscope/benchmarks/_meta/sciq.json | 10 +- .../benchmarks/_meta/screenspot_pro.json | 347 +++ .../benchmarks/_meta/seed_bench_2_plus.json | 10 +- .../evalscope/benchmarks/_meta/simple_qa.json | 7 +- .../benchmarks/_meta/simple_vqa.json | 10 +- .../evalscope/benchmarks/_meta/siqa.json | 10 +- .../benchmarks/_meta/skillsbench.json | 10 +- .../evalscope/benchmarks/_meta/slake.json | 254 ++ .../benchmarks/_meta/super_gpqa.json | 10 +- .../evalscope/benchmarks/_meta/surds.json | 321 ++ .../benchmarks/_meta/swe_bench_lite.json | 10 +- .../_meta/swe_bench_lite_agentic.json | 14 +- .../_meta/swe_bench_multilingual_agentic.json | 14 +- .../benchmarks/_meta/swe_bench_pro.json | 14 +- .../benchmarks/_meta/swe_bench_verified.json | 10 +- .../_meta/swe_bench_verified_agentic.json | 14 +- .../_meta/swe_bench_verified_mini.json | 10 +- .../swe_bench_verified_mini_agentic.json | 14 +- .../benchmarks/_meta/tau2_bench.json | 23 +- .../benchmarks/_meta/tau3_bench.json | 23 +- .../evalscope/benchmarks/_meta/tau_bench.json | 23 +- .../benchmarks/_meta/terminal_bench_v2.json | 32 +- .../benchmarks/_meta/terminal_bench_v2_1.json | 32 +- .../evalscope/benchmarks/_meta/tifa160.json | 11 +- .../evalscope/benchmarks/_meta/tir_bench.json | 14 +- .../benchmarks/_meta/tool_bench.json | 15 +- .../benchmarks/_meta/toolathlon.json | 20 +- .../evalscope/benchmarks/_meta/torgo.json | 7 +- .../evalscope/benchmarks/_meta/trivia_qa.json | 10 +- .../benchmarks/_meta/triviaqa_indic.json | 183 ++ .../benchmarks/_meta/truthful_qa.json | 8 +- .../evalscope/benchmarks/_meta/tvbench.json | 12 +- .../benchmarks/_meta/tweebank_ner.json | 15 +- .../benchmarks/_meta/tweet_ner_7.json | 15 +- .../benchmarks/_meta/videomme_v2.json | 10 +- .../evalscope/benchmarks/_meta/visfactor.json | 168 ++ .../evalscope/benchmarks/_meta/visulogic.json | 10 +- .../benchmarks/_meta/vlms_are_biased.json | 372 +++ .../evalscope/benchmarks/_meta/vqav2.json | 7 +- .../benchmarks/_meta/vstar_bench.json | 10 +- .../evalscope/benchmarks/_meta/vtcbench.json | 233 ++ .../benchmarks/_meta/wide_search.json | 19 +- .../benchmarks/_meta/winogrande.json | 10 +- .../evalscope/benchmarks/_meta/wmt24pp.json | 7 +- .../evalscope/benchmarks/_meta/wnut2017.json | 15 +- .../evalscope/benchmarks/_meta/world_vqa.json | 10 +- .../benchmarks/_meta/zebralogicbench.json | 7 +- .../evalscope/benchmarks/_meta/zerobench.json | 10 +- .../benchmarks/a_okvqa/a_okvqa_adapter.py | 3 +- .../benchmarks/aa_lcr/aa_lcr_adapter.py | 94 +- .../benchmarks/acebench/acebench_adapter.py | 487 +++- .../evalscope/benchmarks/acebench/checker.py | 560 ++++ .../evalscope/benchmarks/acebench/parser.py | 155 + .../evalscope/benchmarks/acebench/prompts.py | 557 ++++ .../evalscope/benchmarks/acebench/rollout.py | 470 +++ .../benchmarks/acebench/scenarios.py | 76 + .../evalscope/benchmarks/acebench/utils.py | 681 +---- .../benchmarks/agieval/agieval_adapter.py | 9 +- .../evalscope/benchmarks/agieval/utils.py | 23 +- .../evalscope/benchmarks/ai2d/ai2d_adapter.py | 3 +- .../evalscope/benchmarks/aime/aime_adapter.py | 93 +- evalscope/evalscope/benchmarks/aime/grader.py | 7 +- .../benchmarks/aime/math_normalize.py | 1 + .../air_bench/air_bench_chat_adapter.py | 283 +- .../air_bench/air_bench_foundation_adapter.py | 49 +- .../evalscope/benchmarks/air_bench/utils.py | 8 +- .../alpaca_eval/alpaca_eval_adapter.py | 121 +- .../evalscope/benchmarks/amc/amc_adapter.py | 13 +- .../evalscope/benchmarks/arc/arc_adapter.py | 5 +- .../benchmarks/arc_agi_2/arc_agi_2_adapter.py | 4 +- .../benchmarks/arc_indic/__init__.py | 1 + .../benchmarks/arc_indic/arc_indic_adapter.py | 81 + .../arena_hard/arena_hard_adapter.py | 200 +- .../evalscope/benchmarks/arena_hard/utils.py | 21 +- .../benchmarks/arxivmath/arxivmath_adapter.py | 7 +- .../arxivrollbench/arxivrollbench_adapter.py | 4 +- .../benchmarks/automation_bench/__init__.py | 3 + .../automation_bench_adapter.py | 226 ++ .../benchmarks/automation_bench/utils.py | 240 ++ .../baby_vision/baby_vision_adapter.py | 7 +- .../evalscope/benchmarks/bbh/bbh_adapter.py | 11 +- .../evalscope/benchmarks/bbh/cot_prompts.py | 92 +- .../benchmarks/bfcl/v3/bfcl_v3_adapter.py | 16 +- .../benchmarks/bfcl/v3/generation.py | 57 +- .../evalscope/benchmarks/bfcl/v3/utils.py | 2 + .../benchmarks/bfcl/v4/bfcl_v4_adapter.py | 30 +- .../evalscope/benchmarks/bfcl/v4/utils.py | 56 +- .../benchmarks/bhasha_bench_multi/__init__.py | 1 + .../bhasha_bench_multi_adapter.py | 168 ++ .../benchmarks/bhashabenchv1/__init__.py | 1 + .../bhashabenchv1/bhashabenchv1_adapter.py | 142 + .../bigcodebench/bigcodebench_adapter.py | 29 +- .../benchmarks/biomix_qa/biomix_qa_adapter.py | 1 - .../benchmarks/blink/blink_adapter.py | 32 +- .../browsecomp/browsecomp_adapter.py | 89 +- .../benchmarks/cc_ocr_v2/__init__.py | 0 .../benchmarks/cc_ocr_v2/cc_ocr_v2_adapter.py | 247 ++ .../benchmarks/cc_ocr_v2/requirements.txt | 6 + .../evalscope/benchmarks/cc_ocr_v2/utils.py | 710 +++++ .../benchmarks/ceval/ceval_adapter.py | 15 +- .../benchmarks/chartqa/chartqa_adapter.py | 4 +- .../benchmarks/charxiv/charxiv_adapter.py | 148 +- .../evalscope/benchmarks/charxiv/utils.py | 18 +- .../chinese_simple_qa/csimple_qa_adapter.py | 79 +- .../benchmarks/cl_bench/cl_bench_adapter.py | 188 +- .../evalscope/benchmarks/cl_bench/utils.py | 13 - .../benchmarks/claw_eval/claw_eval_adapter.py | 34 +- .../evalscope/benchmarks/claw_eval/utils.py | 17 +- .../benchmarks/cmath/cmath_adapter.py | 7 +- .../benchmarks/cmmlu/cmmlu_adapter.py | 14 +- .../benchmarks/cmmmu/cmmmu_adapter.py | 14 +- .../evalscope/benchmarks/cmmu/cmmu_adapter.py | 109 +- .../benchmarks/coin_flip/coin_flip_adapter.py | 26 +- .../common_voice_15_adapter.py | 3 +- .../commonsense_qa/commonsense_qa_adapter.py | 1 - .../competition_math_adapter.py | 14 +- .../evalscope/benchmarks/count_qa/__init__.py | 0 .../benchmarks/count_qa/count_qa_adapter.py | 171 ++ .../data_collection_adapter.py | 62 +- .../benchmarks/deep_swe/deep_swe_adapter.py | 67 +- .../evalscope/benchmarks/deep_swe/utils.py | 4 +- .../benchmarks/deepsearchqa/__init__.py | 1 + .../deepsearchqa/deepsearchqa_adapter.py | 154 + .../benchmarks/deepsearchqa/utils.py | 233 ++ .../benchmarks/docmath/docmath_adapter.py | 88 +- .../evalscope/benchmarks/docmath/utils.py | 4 +- .../benchmarks/docvqa/docvqa_adapter.py | 4 +- .../drivelology/drivelology_binary_adapter.py | 18 +- .../drivelology_multilabel_adapter.py | 38 +- .../drivelology_selection_adapter.py | 1 - .../drivelology_writing_adapter.py | 172 +- .../evalscope/benchmarks/drop/drop_adapter.py | 17 +- evalscope/evalscope/benchmarks/drop/utils.py | 9 +- .../emb_spatial_bench_adapter.py | 2 +- .../benchmarks/eq_bench/answer_validation.py | 2 +- .../benchmarks/eq_bench/eq_bench_adapter.py | 4 +- .../evalscope/benchmarks/erqa/erqa_adapter.py | 16 +- .../benchmarks/fleurs/fleurs_adapter.py | 213 +- .../benchmarks/frames/frames_adapter.py | 90 +- .../evalscope/benchmarks/frames/utils.py | 1 - .../evalscope/benchmarks/gaia/gaia_adapter.py | 1 + evalscope/evalscope/benchmarks/gaia/scorer.py | 2 +- .../benchmarks/gdpval/gdpval_adapter.py | 24 +- .../evalscope/benchmarks/gdpval/utils.py | 14 +- .../general_arena/general_arena_adapter.py | 324 ++- .../benchmarks/general_arena/utils.py | 24 +- .../general_fc/general_fc_adapter.py | 6 +- .../general_mcq/general_mcq_adapter.py | 3 +- .../general_qa/general_qa_adapter.py | 25 +- .../benchmarks/general_qa_vqa_metrics.py | 41 + .../general_vmcq/general_vmcq_adapter.py | 40 +- .../general_vqa/general_vqa_adapter.py | 42 +- .../evalscope/benchmarks/gpqa/gpqa_adapter.py | 18 +- .../benchmarks/gsm8k/gsm8k_adapter.py | 20 +- .../benchmarks/gsm8k_indic/__init__.py | 1 + .../gsm8k_indic/gsm8k_indic_adapter.py | 89 + .../benchmarks/gsm8k_v/gsm8k_v_adapter.py | 11 +- .../hallusion_bench_adapter.py | 50 +- .../benchmarks/halu_eval/halu_eval_adapter.py | 12 +- .../healthbench/healthbench_adapter.py | 143 +- .../evalscope/benchmarks/healthbench/utils.py | 26 +- .../benchmarks/hellaswag/hellaswag_adapter.py | 1 - .../benchmarks/hellaswag_hi/__init__.py | 1 + .../hellaswag_hi/hellaswag_hi_adapter.py | 68 + .../evalscope/benchmarks/hipho/__init__.py | 0 .../benchmarks/hipho/hipho_adapter.py | 420 +++ evalscope/evalscope/benchmarks/hipho/utils.py | 172 ++ .../evalscope/benchmarks/hle/hle_adapter.py | 107 +- .../benchmarks/hmmt/hmmt25_adapter.py | 8 +- .../benchmarks/hmmt/hmmt26_adapter.py | 8 +- .../benchmarks/hmmt/hmmt_nov25_adapter.py | 82 + evalscope/evalscope/benchmarks/hmmt/utils.py | 3 +- .../benchmarks/humaneval/humaneval_adapter.py | 17 +- .../evalscope/benchmarks/humaneval/utils.py | 3 +- .../humanevalplus/humanevalplus_adapter.py | 17 +- .../benchmarks/ifbench/evaluation_lib.py | 1 + .../benchmarks/ifbench/ifbench_adapter.py | 12 +- .../benchmarks/ifbench/instructions.py | 613 ++-- .../benchmarks/ifbench/instructions_util.py | 4 +- .../benchmarks/ifeval/ifeval_adapter.py | 13 +- .../benchmarks/ifeval/instructions.py | 140 +- .../ifeval/instructions_registry.py | 27 +- .../benchmarks/ifeval/instructions_util.py | 7 +- .../image_edit/gedit/gedit_adapter.py | 127 +- .../benchmarks/image_edit/gedit/utils.py | 5 +- .../imo_answerbench_adapter.py | 11 +- .../benchmarks/indic_boolq/__init__.py | 1 + .../indic_boolq/indic_boolq_adapter.py | 87 + .../benchmarks/indic_param/__init__.py | 1 + .../indic_param/indic_param_adapter.py | 98 + .../benchmarks/infovqa/infovqa_adapter.py | 4 +- .../benchmarks/iquiz/iquiz_adapter.py | 1 - .../benchmarks/job_bench/__init__.py | 3 + .../benchmarks/job_bench/job_bench_adapter.py | 339 +++ .../evalscope/benchmarks/job_bench/utils.py | 571 ++++ .../k2_verifier/k2_verifier_adapter.py | 7 +- .../kimi_verifier/kimi_verifier_adapter.py | 11 +- .../benchmarks/kimi_verifier/param_spec.py | 1 + .../evalscope/benchmarks/kina/kina_adapter.py | 2 +- .../librispeech/librispeech_adapter.py | 3 +- .../live_code_bench/evaluate_utils.py | 22 +- .../live_code_bench/extract_utils.py | 4 +- .../live_code_bench_adapter.py | 39 +- .../benchmarks/live_code_bench/load_utils.py | 14 +- .../benchmarks/live_code_bench/prompts.py | 28 +- .../live_code_bench/sandbox_evaluate_utils.py | 13 +- .../live_code_bench/testing_util.py | 10 +- .../benchmarks/locomo/locomo_adapter.py | 31 +- .../benchmarks/logi_qa/logi_qa_adapter.py | 3 +- .../longbench_v2/longbench_v2_adapter.py | 1 - .../longmemeval/longmemeval_adapter.py | 117 +- .../evalscope/benchmarks/longmemeval/utils.py | 35 +- .../maritime_bench/maritime_bench_adapter.py | 2 +- .../maritime_ocr_bench_adapter.py | 13 +- .../spotting_json_quantization_eval.py | 8 +- .../scoring/spotting_quantization_eval.py | 25 +- .../benchmarks/math_500/math_500_adapter.py | 9 +- .../benchmarks/math_qa/math_qa_adapter.py | 1 - .../math_verse/math_verse_adapter.py | 11 +- .../math_vision/math_vision_adapter.py | 29 +- .../math_vista/math_vista_adapter.py | 31 +- .../evalscope/benchmarks/mbpp/mbpp_adapter.py | 16 +- .../benchmarks/mbppplus/mbppplus_adapter.py | 16 +- .../benchmarks/mcp_atlas/mcp_atlas_adapter.py | 142 +- .../evalscope/benchmarks/mcp_atlas/utils.py | 25 +- .../measure_bench/measure_bench_adapter.py | 7 +- .../benchmarks/measure_bench/utils.py | 1 + .../benchmarks/med_mcqa/med_mcqa_adapter.py | 1 - .../benchmarks/medxpertqa/__init__.py | 0 .../medxpertqa/medxpertqa_adapter.py | 119 + .../evalscope/benchmarks/mgsm/mgsm_adapter.py | 18 +- .../benchmarks/mia_bench/mia_bench_adapter.py | 131 +- .../evalscope/benchmarks/mia_bench/utils.py | 109 +- .../benchmarks/micro_vqa/micro_vqa_adapter.py | 1 - .../evalscope/benchmarks/milu/__init__.py | 1 + .../evalscope/benchmarks/milu/milu_adapter.py | 91 + .../minerva_math/minerva_math_adapter.py | 11 +- .../minimax_verifier/_validators.py | 10 +- .../minimax_verifier_adapter.py | 5 +- .../evalscope/benchmarks/miniwob/__init__.py | 1 + .../benchmarks/miniwob/miniwob_adapter.py | 122 + .../benchmarks/miniwob/requirements.txt | 1 + .../evalscope/benchmarks/miniwob/utils.py | 229 ++ .../benchmarks/mm_bench/mm_bench_adapter.py | 12 +- .../benchmarks/mm_star/mm_star_adapter.py | 13 +- .../evalscope/benchmarks/mmau/mmau_adapter.py | 18 +- .../evalscope/benchmarks/mmlu/mmlu_adapter.py | 1 - .../benchmarks/mmlu_pro/mmlu_pro_adapter.py | 17 +- .../mmlu_redux/mmlu_redux_adapter.py | 7 +- .../benchmarks/mmmlu/mmmlu_adapter.py | 1 - .../evalscope/benchmarks/mmmu/mmmu_adapter.py | 20 +- .../benchmarks/mmmu_pro/mmmu_pro_adapter.py | 25 +- .../benchmarks/mri_mcqa/mri_mcqa_adapter.py | 1 - .../benchmarks/msr_vtt/msr_vtt_adapter.py | 14 +- .../evalscope/benchmarks/msvd/msvd_adapter.py | 13 +- .../evalscope/benchmarks/multi_if/ifeval.py | 190 +- .../evalscope/benchmarks/multi_if/metrics.py | 7 +- .../benchmarks/multi_if/multi_if_adapter.py | 32 +- .../multipl_e/multiple_humaneval_adapter.py | 7 +- .../multipl_e/multiple_mbpp_adapter.py | 7 +- .../evalscope/benchmarks/multipl_e/utils.py | 2 +- .../music_trivia/music_trivia_adapter.py | 1 - .../evalscope/benchmarks/musr/musr_adapter.py | 3 +- .../benchmarks/mvbench/mvbench_adapter.py | 3 +- .../evalscope/benchmarks/mvbench/utils.py | 3 +- .../needle_haystack_adapter.py | 132 +- .../benchmarks/needle_haystack/utils.py | 16 +- .../benchmarks/ner/anat_em_adapter.py | 2 +- .../evalscope/benchmarks/ner/bc2gm_adapter.py | 2 +- .../benchmarks/ner/bc4chemd_adapter.py | 2 +- .../benchmarks/ner/bc5cdr_adapter.py | 4 +- .../ner/broad_twitter_corpus_adapter.py | 1 + .../benchmarks/ner/conll2003_adapter.py | 3 +- .../benchmarks/ner/conllpp_adapter.py | 4 +- .../benchmarks/ner/copious_adapter.py | 7 +- .../benchmarks/ner/cross_ner_adapter.py | 9 +- .../benchmarks/ner/cross_ner_entities/ai.py | 48 +- .../ner/cross_ner_entities/literature.py | 17 +- .../ner/cross_ner_entities/music.py | 19 +- .../ner/cross_ner_entities/politics.py | 22 +- .../ner/cross_ner_entities/science.py | 39 +- .../benchmarks/ner/fin_ner_adapter.py | 4 +- .../benchmarks/ner/genia_ner_adapter.py | 8 +- .../benchmarks/ner/harvey_ner_adapter.py | 3 +- .../benchmarks/ner/jnlpba_adapter.py | 6 +- .../benchmarks/ner/jnlpba_rare_adapter.py | 4 +- .../ner/mit_movie_trivia_adapter.py | 7 +- .../benchmarks/ner/mit_restaurant_adapter.py | 3 +- .../benchmarks/ner/multi_nerd_adapter.py | 6 +- .../evalscope/benchmarks/ner/ncbi_adapter.py | 2 +- .../benchmarks/ner/ontonotes5_adapter.py | 5 +- .../benchmarks/ner/tweebank_ner_adapter.py | 4 +- .../benchmarks/ner/tweet_ner_7_adapter.py | 6 +- .../benchmarks/ner/wnut2017_adapter.py | 5 +- .../ocr_bench/ocr_bench/ocr_bench_adapter.py | 22 +- .../ocr_bench/ocr_bench_v2/TEDS_metric.py | 63 +- .../ocr_bench_v2/ocr_bench_v2_adapter.py | 53 +- .../ocr_bench/ocr_bench_v2/page_ocr_metric.py | 1 + .../ocr_bench/ocr_bench_v2/parallel.py | 1 + .../spotting_eval/rrc_evaluation_funcs_1_1.py | 47 +- .../ocr_bench_v2/spotting_eval/script.py | 102 +- .../ocr_bench/ocr_bench_v2/spotting_metric.py | 1 + .../ocr_bench/ocr_bench_v2/utils.py | 60 +- .../benchmarks/officeqa/officeqa_adapter.py | 20 +- .../benchmarks/olmocr_bench/__init__.py | 0 .../olmocr_bench/olmocr_bench_adapter.py | 210 ++ .../benchmarks/olmocr_bench/requirements.txt | 3 + .../benchmarks/olmocr_bench/table_parsing.py | 415 +++ .../benchmarks/olmocr_bench/unit_tests.py | 504 ++++ .../olympiad_bench/olympiad_bench_adapter.py | 15 +- .../benchmarks/olympiad_bench/utils.py | 49 +- .../omni_bench/omni_bench_adapter.py | 11 +- .../omnidoc_bench/legacy/__init__.py | 1 + .../{ => legacy}/end2end_eval.py | 138 +- .../omnidoc_bench/{ => legacy}/metrics.py | 150 +- .../{ => legacy}/omnidoc_bench_adapter.py | 113 +- .../{ => legacy}/requirements.txt | 0 .../omnidoc_bench/{ => legacy}/utils.py | 715 +++-- .../benchmarks/omnidoc_bench/v1_6/__init__.py | 1 + .../v1_6/omnidoc_bench_v1_6_adapter.py | 218 ++ .../omnidoc_bench/v1_6/sandbox_scorer.py | 159 + .../benchmarks/one_million_bench/__init__.py | 0 .../one_million_bench_adapter.py | 318 ++ .../openai_mrcr/openai_mrcr_adapter.py | 30 +- .../evalscope/benchmarks/openai_mrcr/utils.py | 6 +- .../benchmarks/perception_bench/__init__.py | 0 .../perception_bench_adapter.py | 206 ++ .../benchmarks/perception_bench/utils.py | 103 + .../perspective_gap_adapter.py | 12 +- .../evalscope/benchmarks/phyx/__init__.py | 0 .../evalscope/benchmarks/phyx/phyx_adapter.py | 363 +++ evalscope/evalscope/benchmarks/phyx/utils.py | 267 ++ .../evalscope/benchmarks/piqa/piqa_adapter.py | 1 - .../benchmarks/plawbench/__init__.py | 0 .../benchmarks/plawbench/plawbench_adapter.py | 242 ++ .../evalscope/benchmarks/plawbench/utils.py | 636 ++++ .../evalscope/benchmarks/pmc_vqa/__init__.py | 0 .../benchmarks/pmc_vqa/pmc_vqa_adapter.py | 136 + .../benchmarks/poly_math/poly_math_adapter.py | 40 +- .../benchmarks/poly_math/utils/instruction.py | 2 +- .../evalscope/benchmarks/pope/pope_adapter.py | 8 +- .../evalscope/benchmarks/prbench/__init__.py | 0 .../benchmarks/prbench/prbench_adapter.py | 289 ++ .../process_bench/process_bench_adapter.py | 11 +- .../benchmarks/pumed_qa/pubmed_qa_adapter.py | 31 +- .../evalscope/benchmarks/qasc/qasc_adapter.py | 1 - .../evalscope/benchmarks/race/race_adapter.py | 3 +- .../real_world_qa/real_world_qa_adapter.py | 3 +- .../benchmarks/ref_adv_s/__init__.py | 0 .../benchmarks/ref_adv_s/ref_adv_s_adapter.py | 255 ++ .../evalscope/benchmarks/ref_adv_s/utils.py | 140 + .../benchmarks/refcoco/evaluation_lib.py | 12 +- .../benchmarks/refcoco/refcoco_adapter.py | 30 +- .../researchrubrics_adapter.py | 413 +-- .../benchmarks/researchrubrics/utils.py | 103 +- .../benchmarks/sanskriti/__init__.py | 1 + .../benchmarks/sanskriti/sanskriti_adapter.py | 91 + .../benchmarks/scicode/docker/process_data.py | 9 +- .../benchmarks/scicode/docker/test_util.py | 8 +- .../benchmarks/scicode/scicode_adapter.py | 35 +- .../science_qa/science_qa_adapter.py | 1 - .../evalscope/benchmarks/sciq/sciq_adapter.py | 1 - .../benchmarks/screenspot_pro/__init__.py | 0 .../screenspot_pro/screenspot_pro_adapter.py | 167 ++ .../benchmarks/screenspot_pro/utils.py | 159 + .../seed_bench_2_plus_adapter.py | 1 - .../seed_tts_eval/seed_tts_eval_adapter.py | 2 +- .../benchmarks/simple_qa/simple_qa_adapter.py | 73 +- .../simple_vqa/simple_vqa_adapter.py | 85 +- .../evalscope/benchmarks/siqa/siqa_adapter.py | 1 - .../skillsbench/skillsbench_adapter.py | 18 +- .../evalscope/benchmarks/skillsbench/utils.py | 4 +- .../evalscope/benchmarks/slake/__init__.py | 0 .../benchmarks/slake/slake_adapter.py | 175 ++ evalscope/evalscope/benchmarks/slake/utils.py | 118 + .../super_gpqa/super_gpqa_adapter.py | 7 +- .../evalscope/benchmarks/super_gpqa/utils.py | 2 - .../evalscope/benchmarks/surds/__init__.py | 1 + .../benchmarks/surds/surds_adapter.py | 209 ++ evalscope/evalscope/benchmarks/surds/utils.py | 372 +++ .../benchmarks/swe_bench/__init__.py | 6 +- .../benchmarks/swe_bench/build_images.py | 2 +- .../benchmarks/swe_bench/swe_bench_adapter.py | 63 +- .../swe_bench/swe_bench_agentic_adapter.py | 51 +- .../evalscope/benchmarks/swe_bench/utils.py | 6 +- .../swe_bench_pro_agentic_adapter.py | 15 +- .../benchmarks/swe_bench_pro/utils.py | 4 +- .../tau_bench/tau2_bench/generation.py | 8 +- .../tau2_bench/tau2_bench_adapter.py | 27 +- .../tau_bench/tau3_bench/generation.py | 9 +- .../tau3_bench/tau3_bench_adapter.py | 36 +- .../tau_bench/tau_bench/generation.py | 45 +- .../tau_bench/tau_bench/tau_bench_adapter.py | 41 +- .../terminal_bench/terminal_bench_adapter.py | 176 +- .../benchmarks/terminal_bench/utils.py | 15 +- .../benchmarks/text2image/evalmuse_adapter.py | 36 +- .../text2image/genai_bench_adapter.py | 5 +- .../text2image/general_t2i_adapter.py | 1 - .../benchmarks/text2image/hpdv2_adapter.py | 5 +- .../benchmarks/text2image/tifa_adapter.py | 3 +- .../benchmarks/tir_bench/tir_bench_adapter.py | 8 +- .../evalscope/benchmarks/tir_bench/utils.py | 10 +- .../tool_bench/tool_bench_adapter.py | 8 +- .../evalscope/benchmarks/tool_bench/utils.py | 6 +- .../evalscope/benchmarks/toolathlon/client.py | 404 ++- .../toolathlon/toolathlon_adapter.py | 47 +- .../benchmarks/toolathlon/ws_client.py | 5 +- .../benchmarks/torgo/torgo_adapter.py | 9 +- .../benchmarks/trivia_qa/trivia_qa_adapter.py | 12 +- .../benchmarks/triviaqa_indic/__init__.py | 1 + .../triviaqa_indic/triviaqa_indic_adapter.py | 75 + .../truthful_qa/truthful_qa_adapter.py | 6 +- .../benchmarks/tvbench/tvbench_adapter.py | 5 +- .../evalscope/benchmarks/tvbench/utils.py | 4 +- .../evalscope/benchmarks/videomme_v2/utils.py | 3 +- .../videomme_v2/videomme_v2_adapter.py | 3 +- .../benchmarks/visfactor/__init__.py | 0 .../evalscope/benchmarks/visfactor/utils.py | 94 + .../benchmarks/visfactor/visfactor_adapter.py | 155 + .../visu_logic/visu_logic_adapter.py | 9 +- .../benchmarks/vlms_are_biased/__init__.py | 0 .../benchmarks/vlms_are_biased/utils.py | 34 + .../vlms_are_biased_adapter.py | 149 + .../benchmarks/vqav2/vqav2_adapter.py | 2 +- .../vstar_bench/vstar_bench_adapter.py | 1 - .../evalscope/benchmarks/vtcbench/__init__.py | 0 .../benchmarks/vtcbench/requirements.txt | 1 + .../benchmarks/vtcbench/vtcbench_adapter.py | 228 ++ .../wenet_speech/wenet_speech_adapter.py | 7 +- .../evalscope/benchmarks/wide_search/utils.py | 426 +-- .../wide_search/wide_search_adapter.py | 232 +- .../winogrande/winogrande_adapter.py | 1 - .../evalscope/benchmarks/wmt/wmt24_adapter.py | 32 +- .../benchmarks/world_vqa/world_vqa_adapter.py | 5 +- .../benchmarks/zebralogicbench/utils.py | 50 +- .../zebralogicbench_adapter.py | 84 +- .../benchmarks/zerobench/zerobench_adapter.py | 7 +- evalscope/evalscope/cli/benchmark_info.py | 22 +- evalscope/evalscope/cli/start_app.py | 1 + evalscope/evalscope/cli/start_eval.py | 6 +- evalscope/evalscope/cli/start_perf.py | 13 +- evalscope/evalscope/cli/start_service.py | 16 +- evalscope/evalscope/collections/sampler.py | 4 +- evalscope/evalscope/collections/schema.py | 24 +- evalscope/evalscope/config.py | 373 ++- evalscope/evalscope/constants.py | 102 +- evalscope/evalscope/evaluation_versioning.py | 306 ++ evalscope/evalscope/evaluator/__init__.py | 1 + .../evalscope/evaluator/batch_reviewer.py | 20 +- evalscope/evalscope/evaluator/evaluator.py | 147 +- .../evalscope/evaluator/execution_tracker.py | 42 + .../evalscope/evaluator/perf_collector.py | 5 +- evalscope/evalscope/filters/__init__.py | 4 +- evalscope/evalscope/filters/extraction.py | 3 +- evalscope/evalscope/filters/selection.py | 5 +- evalscope/evalscope/metrics/__init__.py | 3 +- .../evalscope/metrics/aggregators/__init__.py | 12 +- .../metrics/aggregators/aggregators.py | 350 ++- evalscope/evalscope/metrics/audio/metrics.py | 41 +- evalscope/evalscope/metrics/judge/__init__.py | 6 - evalscope/evalscope/metrics/judge/base.py | 25 - .../evalscope/metrics/judge/llm_judge.py | 94 +- .../metrics/judge/score_extractors.py | 72 - evalscope/evalscope/metrics/math/parser.py | 65 +- .../metrics/nlp/bert_score/scorer.py | 4 +- .../evalscope/metrics/nlp/bert_score/utils.py | 88 +- evalscope/evalscope/metrics/nlp/metrics.py | 20 +- .../evalscope/metrics/nlp/sem_score/scorer.py | 19 +- .../evalscope/metrics/semantics/__init__.py | 55 + .../evalscope/metrics/semantics/baselines.py | 212 ++ .../evalscope/metrics/semantics/catalog.py | 522 ++++ .../evalscope/metrics/semantics/entry.py | 97 + .../evalscope/metrics/semantics/formatting.py | 255 ++ .../evalscope/metrics/semantics/identity.py | 370 +++ .../evalscope/metrics/semantics/legacy.py | 48 + .../evalscope/metrics/semantics/migration.py | 184 ++ evalscope/evalscope/metrics/semantics/perf.py | 240 ++ .../evalscope/metrics/semantics/ranking.py | 44 + .../evalscope/metrics/semantics/resolver.py | 287 ++ .../evalscope/metrics/semantics/summary.py | 37 + .../utils/bundled_rouge_score/rouge_scorer.py | 5 +- .../evalscope/metrics/utils/functions.py | 7 +- evalscope/evalscope/metrics/utils/rouge.py | 92 +- .../metrics/utils/text_normalizer/basic.py | 16 +- .../metrics/utils/text_normalizer/chinese.py | 32 +- .../metrics/utils/text_normalizer/english.py | 45 +- .../metrics/utils/text_normalizer/wer.py | 4 +- evalscope/evalscope/metrics/vision/metrics.py | 39 +- .../metrics/vision/t2v_metrics/constants.py | 1 + .../build_mps_model/clip_model.py | 5 +- .../models/clipscore_models/clip_model.py | 3 +- .../models/clipscore_models/hpsv2_model.py | 3 +- .../models/clipscore_models/mps_model.py | 3 +- .../clipscore_models/pickscore_model.py | 5 +- .../models/itmscore_models/blip2_itm_model.py | 3 +- .../models/itmscore_models/fga_blip2_model.py | 3 +- .../image_reward/ImageReward.py | 1 + .../itmscore_models/image_reward_model.py | 3 +- .../vision/t2v_metrics/models/model.py | 7 +- .../vision/t2v_metrics/models/utils.py | 1 + .../clip_t5/model/language_model/clip_t5.py | 5 +- .../model/multimodal_projector/builder.py | 1 + .../models/vqascore_models/clip_t5_model.py | 3 +- .../models/vqascore_models/gpt4v_model.py | 3 +- .../models/vqascore_models/lavis/__init__.py | 1 + .../vqascore_models/lavis/common/config.py | 3 +- .../lavis/common/dist_utils.py | 1 + .../vqascore_models/lavis/common/logger.py | 3 +- .../vqascore_models/lavis/common/utils.py | 10 +- .../lavis/common/vqa_tools/vqa.py | 2 +- .../vqascore_models/lavis/models/__init__.py | 1 + .../lavis/models/base_model.py | 3 +- .../lavis/models/blip2_models/Qformer.py | 7 +- .../lavis/models/blip2_models/blip2.py | 2 + .../models/blip2_models/blip2_qformer.py | 1 + .../lavis/models/blip2_models/blip2_t5.py | 1 + .../models/blip2_models/blip2_t5_instruct.py | 1 + .../models/blip2_models/modeling_llama.py | 3 +- .../lavis/models/blip2_models/modeling_t5.py | 8 +- .../lavis/models/blip_models/__init__.py | 3 +- .../lavis/models/blip_models/blip.py | 2 + .../models/blip_models/blip_classification.py | 3 +- .../blip_models/blip_feature_extractor.py | 3 +- .../lavis/models/blip_models/blip_nlvr.py | 1 + .../lavis/models/blip_models/blip_outputs.py | 5 +- .../lavis/models/blip_models/blip_pretrain.py | 3 +- .../lavis/models/blip_models/nlvr_encoder.py | 3 +- .../vqascore_models/lavis/models/clip_vit.py | 5 +- .../vqascore_models/lavis/models/eva_vit.py | 3 +- .../vqascore_models/lavis/models/med.py | 7 +- .../vqascore_models/lavis/models/vit.py | 3 +- .../lavis/processors/blip_processors.py | 1 + .../models/vqascore_models/mm_utils.py | 1 + .../models/vqascore_models/vqa_model.py | 3 +- .../metrics/vision/t2v_metrics/score.py | 5 +- .../evalscope/models/anthropic_compatible.py | 233 +- .../evalscope/models/image_edit_model.py | 15 +- .../evalscope/models/litellm_compatible.py | 181 +- evalscope/evalscope/models/mockllm.py | 10 +- evalscope/evalscope/models/model_apis.py | 18 +- evalscope/evalscope/models/modelscope.py | 58 +- .../evalscope/models/openai_compatible.py | 300 +- .../evalscope/models/openai_responses.py | 98 +- .../evalscope/models/text2image_model.py | 12 +- .../evalscope/models/text2speech_model.py | 33 +- evalscope/evalscope/models/utils/anthropic.py | 106 +- .../evalscope/models/utils/async_client.py | 113 + evalscope/evalscope/models/utils/openai.py | 149 +- .../models/utils/openai_responses.py | 20 +- evalscope/evalscope/perf/arguments.py | 37 +- evalscope/evalscope/perf/benchmark.py | 63 +- evalscope/evalscope/perf/core/__init__.py | 5 +- evalscope/evalscope/perf/core/http_client.py | 62 +- .../evalscope/perf/core/metrics_consumer.py | 45 +- evalscope/evalscope/perf/core/pipeline.py | 50 + .../evalscope/perf/core/strategies/base.py | 17 +- .../perf/core/strategies/closed_loop.py | 153 +- .../perf/core/strategies/multi_turn.py | 138 +- .../perf/core/strategies/open_loop.py | 174 +- evalscope/evalscope/perf/main.py | 21 +- evalscope/evalscope/perf/multi_turn_args.py | 3 +- .../evalscope/perf/multi_turn_benchmark.py | 30 +- evalscope/evalscope/perf/plugin/__init__.py | 4 +- evalscope/evalscope/perf/plugin/api/base.py | 30 +- .../evalscope/perf/plugin/api/custom_api.py | 6 +- .../perf/plugin/api/dashscope_api.py | 3 +- .../evalscope/perf/plugin/api/default_api.py | 40 +- .../evalscope/perf/plugin/api/openai_api.py | 88 +- .../perf/plugin/api/openai_rerank_api.py | 1 + .../perf/plugin/api/openai_responses_api.py | 14 +- .../evalscope/perf/plugin/datasets/base.py | 135 +- .../evalscope/perf/plugin/datasets/custom.py | 3 +- .../perf/plugin/datasets/dataset_args.py | 37 +- .../perf/plugin/datasets/embedding_dataset.py | 8 +- .../perf/plugin/datasets/line_by_line.py | 19 +- .../perf/plugin/datasets/longalpaca.py | 11 +- .../perf/plugin/datasets/multi_turn.py | 17 +- .../evalscope/perf/plugin/datasets/openqa.py | 9 +- .../perf/plugin/datasets/random_dataset.py | 16 +- .../perf/plugin/datasets/random_vl_dataset.py | 3 +- .../perf/plugin/datasets/rerank_dataset.py | 3 +- .../perf/plugin/datasets/share_gpt.py | 26 +- .../perf/plugin/datasets/speed_benchmark.py | 14 +- .../perf/plugin/datasets/swe_smith.py | 33 +- .../evalscope/perf/plugin/datasets/trie.py | 28 +- .../evalscope/perf/plugin/datasets/utils.py | 186 +- .../perf/plugin/datasets/workload_trace.py | 16 +- evalscope/evalscope/perf/plugin/registry.py | 2 + evalscope/evalscope/perf/sla/sla_criterion.py | 8 +- evalscope/evalscope/perf/sla/sla_run.py | 54 +- .../evalscope/perf/utils/analysis_result.py | 2 +- .../evalscope/perf/utils/benchmark_util.py | 94 +- evalscope/evalscope/perf/utils/db_util.py | 114 +- evalscope/evalscope/perf/utils/handler.py | 72 +- .../evalscope/perf/utils/local_server.py | 8 +- evalscope/evalscope/perf/utils/log_utils.py | 5 +- .../evalscope/perf/utils/perf_constants.py | 14 +- evalscope/evalscope/perf/utils/perf_models.py | 142 +- .../perf/utils/report/generate_report.py | 20 +- .../perf/utils/report/perf_charts.py | 130 +- .../evalscope/perf/utils/report/perf_data.py | 34 +- .../evalscope/perf/utils/report/summary.py | 252 +- .../evalscope/perf/utils/rich_display.py | 161 +- .../evalscope/perf/utils/trace_metrics.py | 33 +- .../evalscope/perf/utils/workload_timeline.py | 39 +- evalscope/evalscope/report/__init__.py | 22 +- evalscope/evalscope/report/combinator.py | 149 +- evalscope/evalscope/report/generator.py | 120 +- evalscope/evalscope/report/ref.py | 54 + evalscope/evalscope/report/renderer.py | 141 +- evalscope/evalscope/report/report.py | 282 +- .../evalscope/report/template/css/base.css | 94 +- .../report/template/css/perf_extra.css | 2 +- .../evalscope/report/template/js/i18n_eval.js | 4 + .../evalscope/report/template/js/shared.js | 46 + .../report/template/js/theme_init.js | 19 + .../report/template/perf_report.html.j2 | 3 + .../evalscope/report/template/report.html.j2 | 57 +- evalscope/evalscope/report/visualization.py | 57 +- evalscope/evalscope/run.py | 51 +- .../evalscope/service/api_models/__init__.py | 111 + .../evalscope/service/api_models/common.py | 40 + .../evalscope/service/api_models/eval.py | 43 + .../evalscope/service/api_models/perf.py | 97 + .../evalscope/service/api_models/reports.py | 293 ++ evalscope/evalscope/service/app.py | 83 +- .../evalscope/service/blueprints/eval.py | 56 +- .../evalscope/service/blueprints/perf.py | 115 +- .../evalscope/service/blueprints/reports.py | 807 +++-- evalscope/evalscope/service/perf_archive.py | 135 +- .../evalscope/service/report_meta_cache.py | 130 + evalscope/evalscope/service/responses.py | 13 + evalscope/evalscope/service/utils/__init__.py | 12 +- .../evalscope/service/utils/benchmarks.py | 1 + evalscope/evalscope/service/utils/process.py | 98 +- evalscope/evalscope/summarizer/summarizer.py | 3 +- .../evalscope/third_party/acebench/LICENSE | 25 + .../third_party/acebench/__init__.py | 18 + .../third_party/acebench/en/__init__.py | 1 + .../third_party/acebench/en/base_api.py | 24 + .../third_party/acebench/en/food_services.py | 250 ++ .../third_party/acebench/en/message.py | 284 ++ .../third_party/acebench/en/reminder.py | 142 + .../third_party/acebench/en/travel.py | 528 ++++ .../third_party/acebench/zh/__init__.py | 1 + .../third_party/acebench/zh/base_api.py | 23 + .../third_party/acebench/zh/food_services.py | 245 ++ .../third_party/acebench/zh/message.py | 286 ++ .../third_party/acebench/zh/reminder.py | 176 ++ .../third_party/acebench/zh/travel.py | 546 ++++ .../third_party/longbench_write/eval.py | 7 +- .../third_party/longbench_write/infer.py | 5 +- .../longbench_write/tools/openai_api.py | 5 +- .../evalscope/third_party/thinkbench/eval.py | 7 +- .../third_party/thinkbench/tools/llm.py | 3 +- .../third_party/toolbench_static/eval.py | 1 + .../third_party/toolbench_static/infer.py | 3 +- evalscope/evalscope/utils/argument_utils.py | 34 +- evalscope/evalscope/utils/asyncio_runtime.py | 383 +++ evalscope/evalscope/utils/chat_service.py | 4 +- evalscope/evalscope/utils/code_utils.py | 85 +- evalscope/evalscope/utils/data_utils.py | 286 +- .../evalscope/utils/deprecation_utils.py | 2 +- .../evalscope/utils/doc_utils/__init__.py | 3 +- .../utils/doc_utils/benchmark_stats.py | 14 +- .../utils/doc_utils/generate_dataset_md.py | 94 +- .../utils/doc_utils/readme_generator.py | 29 +- .../utils/doc_utils/translate_description.py | 15 +- evalscope/evalscope/utils/download_utils.py | 148 + evalscope/evalscope/utils/function_utils.py | 284 +- evalscope/evalscope/utils/io_utils.py | 141 +- evalscope/evalscope/utils/json_schema.py | 7 +- evalscope/evalscope/utils/logger.py | 29 +- evalscope/evalscope/utils/media_utils.py | 103 + evalscope/evalscope/utils/model_utils.py | 17 +- evalscope/evalscope/utils/multi_choices.py | 159 +- evalscope/evalscope/utils/ner.py | 6 +- evalscope/evalscope/utils/resource_utils.py | 94 +- .../utils/tqdm_utils/progress_tracker.py | 2 +- .../utils/tqdm_utils/tqdm_logging.py | 13 +- evalscope/evalscope/utils/uri_utils.py | 61 + evalscope/evalscope/utils/url_utils.py | 174 -- evalscope/evalscope/version.py | 4 +- evalscope/evalscope/web/README.md | 27 +- evalscope/evalscope/web/eslint.config.js | 7 +- evalscope/evalscope/web/package-lock.json | 124 +- evalscope/evalscope/web/package.json | 6 +- .../drift/localeKeyCheck.test.ts | 8 +- .../web/scripts/generateApiContracts.ts | 54 + evalscope/evalscope/web/src/App.tsx | 6 +- .../evalscope/web/src/api/client.test.ts | 43 + evalscope/evalscope/web/src/api/client.ts | 54 +- evalscope/evalscope/web/src/api/errors.ts | 2 +- evalscope/evalscope/web/src/api/eval.ts | 3 +- .../src/api/generated/contracts.schema.json | 2585 +++++++++++++++++ .../web/src/api/generated/contracts.ts | 570 ++++ evalscope/evalscope/web/src/api/perf.ts | 28 +- evalscope/evalscope/web/src/api/reports.ts | 82 +- .../web/src/api/schemas/common.schema.ts | 26 - .../src/api/schemas/endpointSchemas.test.ts | 125 - .../web/src/api/schemas/eval.schema.ts | 72 - .../evalscope/web/src/api/schemas/index.ts | 11 - .../web/src/api/schemas/perf.schema.ts | 110 - .../src/api/schemas/reports.schema.test.ts | 26 - .../web/src/api/schemas/reports.schema.ts | 269 -- .../src/api/schemas/schemas.property.test.ts | 241 -- evalscope/evalscope/web/src/api/task.test.ts | 4 - evalscope/evalscope/web/src/api/task.ts | 15 +- .../evalscope/web/src/api/types/common.ts | 7 +- evalscope/evalscope/web/src/api/types/eval.ts | 5 +- .../evalscope/web/src/api/types/index.ts | 4 +- evalscope/evalscope/web/src/api/types/perf.ts | 6 +- .../evalscope/web/src/api/types/reports.ts | 15 +- .../src/components/a11yResponsive.test.tsx | 5 +- .../web/src/components/charts/MetricTrend.tsx | 169 ++ .../components/chat/AgentTraceView.test.tsx | 72 + .../{single => }/chat/AgentTraceView.tsx | 305 +- .../web/src/components/chat/ChatView.test.tsx | 166 ++ .../components/{single => chat}/ChatView.tsx | 25 +- .../chat/EvalResultPanel.test.tsx | 27 +- .../{single => }/chat/EvalResultPanel.tsx | 46 +- .../components/chat/JudgeReviewPanel.test.tsx | 208 ++ .../src/components/chat/JudgeReviewPanel.tsx | 270 ++ .../{single => }/chat/MediaBlocks.tsx | 26 +- .../{single => }/chat/MessageComponents.tsx | 8 +- .../{single => }/chat/ToolCallComponents.tsx | 14 +- .../{single => }/chat/roleConfig.ts | 11 +- .../web/src/components/common/DataTable.tsx | 16 +- .../components/common/DataTableFallback.tsx | 10 +- .../components/common/ErrorBoundary.test.tsx | 92 + .../src/components/common/ErrorBoundary.tsx | 83 +- .../src/components/common/LoadingSpinner.tsx | 8 - .../common/MarkdownRenderer.test.tsx | 35 - .../components/compare/CompareReportRail.tsx | 157 + .../compare/PredictionCompareTab.tsx | 347 +++ .../src/components/compare/ScoreMatrixTab.tsx | 254 ++ .../src/components/compare/compareSlots.ts | 58 + .../dashboard/AggregatedResults.tsx | 410 +++ .../components/dashboard/dashboard.test.tsx | 240 ++ .../components/eval/EvalConfigForm.test.tsx | 106 + .../src/components/eval/EvalConfigForm.tsx | 416 +-- .../web/src/components/eval/TaskMonitor.tsx | 2 +- .../src/components/perf/PerfChartGroup.tsx | 2 +- .../src/components/perf/PerfConfigForm.tsx | 218 +- .../perf}/PerfRunsTab.tsx | 140 +- .../web/src/components/reports/DatasetNav.tsx | 5 +- .../components/reports/DetailsTab.test.tsx | 81 +- .../web/src/components/reports/DetailsTab.tsx | 143 +- .../components/reports/OverviewTab.test.tsx | 85 +- .../src/components/reports/OverviewTab.tsx | 119 +- .../reports/PerfMetricsPanel.test.tsx | 31 +- .../components/reports/PerfMetricsPanel.tsx | 138 +- .../src/components/reports/PredictionsTab.tsx | 269 +- .../web/src/components/reports/ReportCard.tsx | 52 +- .../src/components/reports/ReportFilters.tsx | 43 +- .../src/components/reports/ReportHeader.tsx | 29 +- .../components/reports/ReportSummaryStats.tsx | 50 +- .../src/components/reports/ReportsTable.tsx | 90 +- .../components/reports/SampleNavigator.tsx | 79 + .../reports/SelectionControls.test.tsx | 35 +- .../src/components/reports/SelectionTray.tsx | 38 +- .../components/reports/metricCells.test.tsx | 165 ++ .../src/components/reports/metricCells.tsx | 137 + .../reports/metricConsistency.test.tsx | 169 +- .../src/components/single/ChatView.test.tsx | 52 - .../tasks/EvalTaskPanel.tsx} | 16 +- .../tasks/PerfTaskPanel.tsx} | 14 +- .../src/components/tasks/TaskFormShell.tsx | 80 + ...Page.test.tsx => TaskRunnerShell.test.tsx} | 6 +- ...TaskRunnerPage.tsx => TaskRunnerShell.tsx} | 12 +- .../tasks/taskFormContract.test.tsx | 174 ++ .../web/src/components/tasks/useTaskForm.ts | 82 + .../web/src/components/ui/Callout.tsx | 71 + .../web/src/components/ui/ChatBubble.tsx | 9 +- .../src/components/ui/ConfirmDialog.test.tsx | 59 + .../web/src/components/ui/ConfirmDialog.tsx | 135 + .../web/src/components/ui/ErrorAlert.tsx | 19 +- .../{common => ui}/ImageLightbox.tsx | 0 .../components/{common => ui}/JsonViewer.tsx | 0 .../web/src/components/ui/KpiCard.tsx | 52 - .../web/src/components/ui/KpiStrip.tsx | 136 + .../web/src/components/ui/LabelledField.tsx | 29 + .../{common => ui}/LazyCodeBlock.tsx | 0 .../components/{common => ui}/LogViewer.tsx | 0 .../components/ui/MarkdownRenderer.test.tsx | 88 + .../{common => ui}/MarkdownRenderer.tsx | 5 +- .../web/src/components/ui/ScoreBadge.tsx | 42 +- .../web/src/components/ui/ScoreBar.test.tsx | 75 + .../web/src/components/ui/ScoreBar.tsx | 89 + .../src/components/ui/ScoreThresholdInput.tsx | 43 + .../src/components/ui/SegmentedControl.tsx | 90 + .../web/src/components/ui/Select.tsx | 4 + .../src/components/ui/SelectionCheckbox.tsx | 4 + .../web/src/components/ui/Tabs.test.tsx | 19 +- .../evalscope/web/src/components/ui/Tabs.tsx | 86 +- .../web/src/components/ui/Tooltip.test.tsx | 78 + .../web/src/components/ui/Tooltip.tsx | 86 + .../components/{common => ui}/VirtualList.tsx | 0 .../{common => ui}/virtualWindow.test.ts | 0 .../{common => ui}/virtualWindow.ts | 0 .../web/src/contexts/LocaleContext.tsx | 10 +- .../web/src/contexts/ReportsContext.test.tsx | 96 + .../web/src/contexts/ReportsContext.tsx | 389 ++- .../web/src/domain/chat/judgeReview.test.ts | 156 + .../web/src/domain/chat/judgeReview.ts | 142 + .../web/src/domain/chat/messageText.test.ts | 111 + .../chat/messageText.ts} | 20 +- .../src/domain/compare/scoreMatrix.test.ts | 134 + .../web/src/domain/compare/scoreMatrix.ts | 111 + ...compareModel.test.ts => selection.test.ts} | 207 +- .../compare/{compareModel.ts => selection.ts} | 170 +- .../form/validation.datasetArgs.test.ts | 6 +- .../form/validation.firstInvalid.test.ts | 6 +- .../domain/form/validation.numeric.test.ts | 6 +- .../web/src/domain/form/validation.ts | 64 + .../src/domain/metric/MetricDisplaySpec.ts | 97 - .../web/src/domain/metric/MetricSemantics.ts | 7 + .../web/src/domain/metric/__arbitraries__.ts | 47 + .../evalscope/web/src/domain/metric/index.ts | 31 + .../src/domain/metric/metricFormat.test.ts | 654 ++--- .../web/src/domain/metric/metricFormat.ts | 393 ++- .../web/src/domain/metric/registry.test.ts | 114 - .../web/src/domain/metric/registry.ts | 396 --- .../domain/metric/staticConstraints.test.ts | 89 + .../{utils/perf.ts => domain/perf/charts.ts} | 10 +- .../web/src/domain/perf/compareModel.test.ts | 793 ----- .../web/src/domain/perf/deltaModel.test.ts | 213 ++ .../perf/{compareModel.ts => deltaModel.ts} | 315 +- .../web/src/domain/perf/perfWorkload.test.ts | 6 +- .../web/src/domain/perf/perfWorkload.ts | 4 +- .../domain/perf/providerResolution.test.ts | 6 +- .../web/src/domain/report/nativeScale.test.ts | 75 + .../web/src/domain/report/primaryMetrics.ts | 83 + .../web/src/domain/report/reportRef.test.ts | 34 + .../web/src/domain/report/reportRef.ts | 46 + .../src/domain/report/runAggregation.test.ts | 345 +++ .../web/src/domain/report/runAggregation.ts | 297 ++ .../web/src/domain/trace/stepGroups.ts | 293 ++ .../domain/trace/textualBlockTrace.test.ts | 131 + .../web/src/domain/trace/toolLinking.test.ts | 179 ++ .../web/src/hooks/useAsyncResource.test.tsx | 149 + .../web/src/hooks/useAsyncResource.ts | 106 + .../web/src/hooks/useBatchDelete.test.tsx | 59 + .../evalscope/web/src/hooks/useBatchDelete.ts | 61 + .../evalscope/web/src/hooks/useQueryParams.ts | 18 +- .../web/src/hooks/useScopedState.test.tsx | 105 + .../evalscope/web/src/hooks/useScopedState.ts | 64 + .../web/src/i18n/keyCoverage.test.ts | 180 ++ .../web/src/i18n/translations/benchmarks.ts | 2 + .../web/src/i18n/translations/common.ts | 6 + .../web/src/i18n/translations/compare.ts | 32 +- .../web/src/i18n/translations/dashboard.ts | 128 +- .../web/src/i18n/translations/eval.ts | 12 + .../web/src/i18n/translations/index.ts | 3 - .../web/src/i18n/translations/metrics.ts | 41 +- .../web/src/i18n/translations/perf.ts | 217 +- .../web/src/i18n/translations/performance.ts | 143 - .../web/src/i18n/translations/prediction.ts | 42 + .../web/src/i18n/translations/reports.ts | 12 +- .../web/src/i18n/translations/trace.ts | 28 +- evalscope/evalscope/web/src/index.css | 48 +- .../evalscope/web/src/layouts/MainLayout.tsx | 29 +- .../web/src/pages/BenchmarksPage.test.tsx | 91 + .../web/src/pages/BenchmarksPage.tsx | 88 +- .../web/src/pages/ComparePage.test.tsx | 230 ++ .../evalscope/web/src/pages/ComparePage.tsx | 1119 +++---- .../evalscope/web/src/pages/DashboardPage.tsx | 520 ++-- .../web/src/pages/PerfComparePage.test.tsx | 60 +- .../web/src/pages/PerfComparePage.tsx | 215 +- .../web/src/pages/PerfReportDetailPage.tsx | 162 +- .../web/src/pages/PerfReportsPage.tsx | 215 +- .../web/src/pages/ReportDetailPage.test.tsx | 78 + .../web/src/pages/ReportDetailPage.tsx | 132 +- .../web/src/pages/ReportViewerPage.test.tsx | 53 + .../web/src/pages/ReportsPage.test.tsx | 114 + .../evalscope/web/src/pages/ReportsPage.tsx | 233 +- .../web/src/pages/TasksPage.test.tsx | 52 + .../evalscope/web/src/pages/TasksPage.tsx | 8 +- .../web/src/pages/dashboardFilter.test.tsx | 257 ++ .../web/src/pages/performanceView.test.tsx | 17 +- .../web/src/test/fixtures/perf-detail.json | 94 +- .../test/fixtures/report-multi-dataset.json | 121 +- .../fixtures/report-real-single-sample.json | 68 +- .../evalscope/web/src/utils/formatUtils.ts | 19 + .../evalscope/web/src/utils/media.test.ts | 43 + evalscope/evalscope/web/src/utils/media.ts | 44 + .../evalscope/web/src/utils/reportParser.ts | 35 - evalscope/evalscope/web/tsconfig.json | 3 +- evalscope/evalscope/web/tsconfig.test.json | 13 + evalscope/evalscope/web/vitest.config.ts | 15 +- evalscope/examples/aigc/image_edit.py | 22 +- evalscope/examples/aigc/image_edit_local.py | 22 +- .../viz/qwen-plus/configs/task_config.yaml | 6 +- .../viz/qwen3-max/configs/task_config.yaml | 6 +- evalscope/pyproject.toml | 43 +- evalscope/requirements/dev.txt | 2 + evalscope/requirements/framework.txt | 1 + evalscope/requirements/rag.txt | 2 +- evalscope/requirements/sandbox.txt | 2 +- .../scripts/generate_web_api_contracts.py | 43 + evalscope/scripts/release/verify_package.py | 97 + evalscope/setup.cfg | 32 - .../external/test_agent_loop_external.py | 9 +- .../test_anthropic_bridge_prompt_cache.py | 169 ++ .../agent/external/test_bridge_lifecycle.py | 363 +++ .../tests/agent/external/test_codex_runner.py | 5 +- .../tests/agent/external/test_docker_e2e.py | 5 +- .../agent/external/test_evaluator_pipeline.py | 13 +- .../agent/external/test_helpers_patch.py | 3 +- .../external/test_openai_chat_multi_turn.py | 5 +- .../external/test_openai_chat_round_trip.py | 5 +- .../test_openai_chat_tool_calls_streaming.py | 5 +- .../agent/external/test_opencode_runner.py | 20 +- .../external/test_qwen_via_claude_code.py | 5 +- .../external/test_responses_error_paths.py | 5 +- .../external/test_responses_event_sequence.py | 5 +- .../test_responses_extended_item_types.py | 1 + .../external/test_responses_multi_turn.py | 5 +- ..._responses_previous_response_id_ignored.py | 5 +- .../external/test_responses_round_trip.py | 5 +- .../test_responses_tool_calls_streaming.py | 5 +- .../external/test_runner_skill_install.py | 89 + .../agent/external/test_walking_skeleton.py | 14 +- evalscope/tests/agent/mcp_echo_server.py | 1 + .../tests/agent/test_agent_integration.py | 222 +- .../tests/agent/test_agent_interfaces.py | 72 +- evalscope/tests/agent/test_agent_loop.py | 457 ++- evalscope/tests/agent/test_docker_image.py | 16 + evalscope/tests/agent/test_native_runner.py | 310 ++ evalscope/tests/agent/test_sandbox_service.py | 373 ++- evalscope/tests/agent/test_skill_runtime.py | 108 + .../tests/agent/test_strategy_parsers.py | 139 + evalscope/tests/agent/test_t2_environment.py | 448 ++- .../test_aggregation_excludes_unusable.py | 131 + evalscope/tests/api/judge/test_contracts.py | 237 ++ evalscope/tests/api/judge/test_executor.py | 440 +++ evalscope/tests/api/judge/test_gates.py | 206 ++ .../tests/api/judge/test_judge_config.py | 155 + .../tests/api/judge/test_llm_judge_mixin.py | 95 + evalscope/tests/api/judge/test_llm_recall.py | 61 + .../tests/api/judge/test_migrated_adapters.py | 319 ++ evalscope/tests/api/test_benchmark_meta.py | 96 + evalscope/tests/api/test_dataset_builder.py | 141 + .../tests/api/test_dataset_dict_limit.py | 70 + evalscope/tests/api/test_dataset_hub.py | 118 + evalscope/tests/api/test_dataset_loader.py | 204 ++ .../tests/api/test_default_data_adapter.py | 213 ++ .../tests/api/test_evaluation_versioning.py | 293 ++ evalscope/tests/api/test_gdpval_benchmark.py | 71 +- evalscope/tests/api/test_io_utils.py | 215 ++ .../tests/api/test_mcp_atlas_benchmark.py | 112 +- .../tests/api/test_messages_to_markdown.py | 60 + evalscope/tests/api/test_request_stats.py | 58 + evalscope/tests/api/test_scoring_policy.py | 101 + .../tests/api/test_seed_tts_eval_benchmark.py | 3 +- .../tests/api/test_shuffle_determinism.py | 75 + .../tests/api/test_task_config_validation.py | 243 ++ evalscope/tests/api/test_text2speech_model.py | 2 +- .../tests/api/test_vision_language_adapter.py | 352 +++ evalscope/tests/benchmark/test_acebench.py | 866 ++++++ evalscope/tests/benchmark/test_agent.py | 1099 ++++++- evalscope/tests/benchmark/test_agent_loop.py | 21 +- evalscope/tests/benchmark/test_asr_filters.py | 85 - evalscope/tests/benchmark/test_bbh_prompts.py | 18 + evalscope/tests/benchmark/test_bfcl_v4.py | 132 + evalscope/tests/benchmark/test_cc_ocr_v2.py | 202 ++ .../tests/benchmark/test_cmmmu_adapter.py | 81 + .../benchmark/test_code_execution_timeouts.py | 51 + evalscope/tests/benchmark/test_count_qa.py | 139 + evalscope/tests/benchmark/test_deep_swe.py | 363 +++ .../tests/benchmark/test_deepsearchqa.py | 126 + evalscope/tests/benchmark/test_eval.py | 313 +- .../benchmark/test_general_qa_adapter.py | 140 + .../benchmark/test_general_vmcq_adapter.py | 198 ++ .../benchmark/test_general_vqa_adapter.py | 177 ++ .../tests/benchmark/test_gpqa_determinism.py | 62 + .../benchmark/test_ifbench_instructions.py | 59 + .../test_ifeval_language_checkers.py | 52 + evalscope/tests/benchmark/test_image_edit.py | 11 +- .../benchmark/test_judge_output_parsing.py | 115 + .../benchmark/test_live_code_bench_prompt.py | 51 + evalscope/tests/benchmark/test_logic_vista.py | 120 + evalscope/tests/benchmark/test_miniwob.py | 142 + evalscope/tests/benchmark/test_models.py | 11 +- .../tests/benchmark/test_olmocr_bench.py | 354 +++ .../tests/benchmark/test_omnidoc_bench.py | 132 + .../tests/benchmark/test_one_million_bench.py | 214 ++ evalscope/tests/benchmark/test_phyx.py | 280 ++ evalscope/tests/benchmark/test_prbench.py | 175 ++ .../tests/benchmark/test_process_bench.py | 30 + evalscope/tests/benchmark/test_ref_adv_s.py | 145 + .../tests/benchmark/test_researchrubrics.py | 343 +++ evalscope/tests/benchmark/test_sandbox.py | 8 +- .../tests/benchmark/test_screenspot_pro.py | 105 + evalscope/tests/benchmark/test_skillsbench.py | 277 ++ evalscope/tests/benchmark/test_slake.py | 86 + evalscope/tests/benchmark/test_surds.py | 206 ++ .../tests/benchmark/test_terminal_bench.py | 147 + .../tests/benchmark/test_toolathlon_client.py | 1005 +++++++ .../tests/benchmark/test_toolathlon_remote.py | 45 + evalscope/tests/benchmark/test_visfactor.py | 120 + evalscope/tests/benchmark/test_vlm.py | 208 +- .../tests/benchmark/test_vlms_are_biased.py | 119 + evalscope/tests/benchmark/test_vtcbench.py | 107 + evalscope/tests/benchmark/test_wide_search.py | 303 ++ evalscope/tests/cli/test_all.py | 23 +- evalscope/tests/cli/test_collection.py | 6 +- evalscope/tests/cli/test_custom.py | 31 +- .../tests/cli/test_metric_semantics_e2e.py | 216 ++ evalscope/tests/cli/test_reasoning.py | 8 +- evalscope/tests/cli/test_service.py | 16 + evalscope/tests/common.py | 8 +- .../tests/evaluator/test_cache_manager.py | 171 ++ .../evaluator/test_execution_completeness.py | 131 + .../aggregators/test_missing_metric_keys.py | 167 ++ .../metrics/aggregators/test_weighted_mean.py | 236 ++ evalscope/tests/metrics/test_nlp_metrics.py | 36 + evalscope/tests/metrics/test_rouge.py | 76 + .../models/test_anthropic_prompt_cache.py | 354 +++ .../models/test_anthropic_stream_retry.py | 124 + .../models/test_anthropic_tool_id_sanitize.py | 139 + .../models/test_async_client_lifecycle.py | 288 ++ .../models/test_empty_completion_retry.py | 200 ++ ...test_generation_config_extra_validation.py | 35 + .../tests/models/test_image_model_output.py | 138 + .../models/test_litellm_reasoning_history.py | 102 + evalscope/tests/models/test_model_cache.py | 60 + .../tests/models/test_modelscope_usage.py | 79 + .../tests/models/test_no_retry_exceptions.py | 308 ++ .../tests/models/test_openai_audio_input.py | 168 ++ .../tests/models/test_openai_reasoning.py | 9 +- .../test_openai_stream_finish_reason.py | 62 + .../models/test_openai_stream_reasoning.py | 90 + .../tests/models/test_openai_stream_retry.py | 100 + evalscope/tests/perf/perf_test_base.py | 3 +- .../tests/perf/test_arguments_secrets.py | 161 + .../tests/perf/test_arguments_validation.py | 72 + evalscope/tests/perf/test_async_lifecycle.py | 406 +++ .../tests/perf/test_chat_template_error.py | 88 + .../perf/test_closed_loop_warmup_handoff.py | 320 ++ evalscope/tests/perf/test_dataset_args.py | 138 + .../tests/perf/test_line_by_line_body.py | 124 + evalscope/tests/perf/test_metrics_consumer.py | 125 + .../perf/test_multi_turn_warmup_handoff.py | 223 ++ .../tests/perf/test_open_loop_strategy.py | 61 + .../perf/test_openai_api_parse_responses.py | 95 + .../tests/perf/test_percentile_metrics.py | 55 + evalscope/tests/perf/test_perf_archive.py | 271 ++ evalscope/tests/perf/test_perf_basic.py | 25 +- .../tests/perf/test_perf_embedding_rerank.py | 12 +- .../tests/perf/test_perf_report_loader.py | 80 + evalscope/tests/perf/test_perf_streaming.py | 118 + evalscope/tests/perf/test_prefix_injection.py | 316 ++ .../tests/perf/test_request_build_errors.py | 102 + .../tests/perf/test_request_generation.py | 93 +- evalscope/tests/perf/test_rich_display.py | 72 +- evalscope/tests/perf/test_stream_metrics.py | 236 ++ evalscope/tests/perf/test_target_input_len.py | 136 + .../tests/perf/test_wall_time_failures.py | 179 ++ evalscope/tests/perf/test_workload_trace.py | 632 ++++ .../tests/perf/test_workload_trace_e2e.py | 293 ++ evalscope/tests/rag/test_backend_manager.py | 74 + evalscope/tests/rag/test_clip_benchmark.py | 3 +- evalscope/tests/rag/test_mteb.py | 3 +- evalscope/tests/rag/test_ragas.py | 1 + .../report/semantics/golden_samples.json | 607 ++++ .../tests/report/semantics/test_baselines.py | 111 + .../semantics/test_benchmark_legacy_rules.py | 140 + .../test_builtin_report_contracts.py | 174 ++ .../tests/report/semantics/test_catalog.py | 75 + .../semantics/test_complex_aggregations.py | 78 + .../tests/report/semantics/test_formatting.py | 248 ++ .../report/semantics/test_golden_samples.py | 195 ++ .../tests/report/semantics/test_naming.py | 124 + .../report/semantics/test_perf_semantics.py | 203 ++ .../tests/report/semantics/test_public_api.py | 40 + .../tests/report/semantics/test_ranking.py | 15 + .../semantics/test_registry_alignment.py | 217 ++ .../report/semantics/test_report_model.py | 552 ++++ .../tests/report/semantics/test_resolver.py | 93 + .../report/semantics/test_semantics_model.py | 60 + .../semantics/test_structured_aggregators.py | 96 + evalscope/tests/report/test_combinator.py | 170 +- .../tests/report/test_execution_summary.py | 37 + .../report/test_prediction_media_paths.py | 51 + .../tests/report/test_renderer_semantics.py | 40 + evalscope/tests/report/test_report_delete.py | 96 + .../tests/report/test_report_endpoints.py | 270 ++ .../tests/report/test_report_meta_cache.py | 75 + evalscope/tests/report/test_report_ref.py | 41 + evalscope/tests/report/test_service_charts.py | 190 ++ .../tests/report/test_static_report_theme.py | 68 + evalscope/tests/service/__init__.py | 0 evalscope/tests/service/test_api_contracts.py | 428 +++ .../tests/service/test_eval_semantic_table.py | 36 + evalscope/tests/service/test_process_stop.py | 84 + evalscope/tests/swift/test_run_swift_eval.py | 3 +- .../tests/swift/test_run_swift_vlm_eval.py | 3 +- .../swift/test_run_swift_vlm_jugde_eval.py | 3 +- evalscope/tests/test_async_generate.py | 177 -- evalscope/tests/test_download_utils.py | 113 + evalscope/tests/test_function_utils.py | 589 ++++ evalscope/tests/test_loader_shuffle.py | 70 - evalscope/tests/test_logger.py | 17 + evalscope/tests/test_multi_choices.py | 274 ++ evalscope/tests/test_openai_responses.py | 265 -- evalscope/tests/test_resource_utils.py | 142 + evalscope/tests/test_run_all.py | 2 +- patches/local-uncommitted-evalscope.patch | 895 ++++++ .../evalscope/api/evaluator/cache.py.patch | 29 + .../evalscope/api/messages/__init__.py.patch | 8 + .../api/model/generate_config.py.patch | 12 + .../local/evalscope/api/model/model.py.patch | 45 + .../deep_swe/deep_swe_adapter.py.patch | 98 + .../benchmarks/swe_bench/utils.py.patch | 35 + .../terminal_bench_adapter.py.patch | 64 + .../evalscope/evaluator/evaluator.py.patch | 38 + .../models/anthropic_compatible.py.patch | 160 + .../models/litellm_compatible.py.patch | 155 + .../models/openai_compatible.py.patch | 208 ++ .../models/openai_responses.py.patch | 104 + .../evalscope/models/utils/openai.py.patch | 117 + .../evalscope/report/combinator.py.patch | 26 + .../evalscope/utils/function_utils.py.patch | 54 + patches/local/request_stats.py | 125 + patches/local/tests/test_request_stats.py | 58 + patches/upstream-base.txt | 2 + patches/upstream.sha | 1 + 2076 files changed, 120676 insertions(+), 25528 deletions(-) create mode 100644 evalscope/.github/workflows/code-review.yaml create mode 100644 evalscope/.github/workflows/frontend.yml create mode 100644 evalscope/custom_eval/multimodal/vqa/example_placeholder.jsonl create mode 100644 evalscope/docs/en/benchmarks/agieval.md create mode 100644 evalscope/docs/en/benchmarks/arc_agi_2.md create mode 100644 evalscope/docs/en/benchmarks/arc_indic.md create mode 100644 evalscope/docs/en/benchmarks/automation_bench.md create mode 100644 evalscope/docs/en/benchmarks/bhasha_bench_multi_ayur.md create mode 100644 evalscope/docs/en/benchmarks/bhasha_bench_multi_finance.md create mode 100644 evalscope/docs/en/benchmarks/bhasha_bench_multi_krishi.md create mode 100644 evalscope/docs/en/benchmarks/bhasha_bench_multi_legal.md create mode 100644 evalscope/docs/en/benchmarks/bhashabenchv1_ayur.md create mode 100644 evalscope/docs/en/benchmarks/bhashabenchv1_finance.md create mode 100644 evalscope/docs/en/benchmarks/bhashabenchv1_krishi.md create mode 100644 evalscope/docs/en/benchmarks/bhashabenchv1_legal.md create mode 100644 evalscope/docs/en/benchmarks/cc_ocr_v2.md create mode 100644 evalscope/docs/en/benchmarks/claw_eval.md create mode 100644 evalscope/docs/en/benchmarks/count_qa.md create mode 100644 evalscope/docs/en/benchmarks/deep_swe.md create mode 100644 evalscope/docs/en/benchmarks/deepsearchqa.md create mode 100644 evalscope/docs/en/benchmarks/emb_spatial_bench.md create mode 100644 evalscope/docs/en/benchmarks/gsm8k_indic.md create mode 100644 evalscope/docs/en/benchmarks/hellaswag_hi.md create mode 100644 evalscope/docs/en/benchmarks/hipho.md create mode 100644 evalscope/docs/en/benchmarks/hmmt_nov25.md create mode 100644 evalscope/docs/en/benchmarks/indic_boolq.md create mode 100644 evalscope/docs/en/benchmarks/indic_param.md create mode 100644 evalscope/docs/en/benchmarks/job_bench.md create mode 100644 evalscope/docs/en/benchmarks/kina.md create mode 100644 evalscope/docs/en/benchmarks/measure_bench.md create mode 100644 evalscope/docs/en/benchmarks/medxpertqa.md create mode 100644 evalscope/docs/en/benchmarks/milu.md create mode 100644 evalscope/docs/en/benchmarks/miniwob.md create mode 100644 evalscope/docs/en/benchmarks/officeqa.md create mode 100644 evalscope/docs/en/benchmarks/olmocr_bench.md create mode 100644 evalscope/docs/en/benchmarks/omni_doc_bench_v1_6.md create mode 100644 evalscope/docs/en/benchmarks/one_million_bench.md create mode 100644 evalscope/docs/en/benchmarks/perception_bench.md create mode 100644 evalscope/docs/en/benchmarks/perspective_gap_prompt_writing.md create mode 100644 evalscope/docs/en/benchmarks/perspective_gap_role_assignment.md create mode 100644 evalscope/docs/en/benchmarks/phyx_mc.md create mode 100644 evalscope/docs/en/benchmarks/phyx_oe.md create mode 100644 evalscope/docs/en/benchmarks/plawbench.md create mode 100644 evalscope/docs/en/benchmarks/pmc_vqa.md create mode 100644 evalscope/docs/en/benchmarks/prbench.md create mode 100644 evalscope/docs/en/benchmarks/ref_adv_s.md create mode 100644 evalscope/docs/en/benchmarks/researchrubrics.md create mode 100644 evalscope/docs/en/benchmarks/sanskriti.md create mode 100644 evalscope/docs/en/benchmarks/screenspot_pro.md create mode 100644 evalscope/docs/en/benchmarks/skillsbench.md create mode 100644 evalscope/docs/en/benchmarks/slake.md create mode 100644 evalscope/docs/en/benchmarks/surds.md create mode 100644 evalscope/docs/en/benchmarks/toolathlon.md create mode 100644 evalscope/docs/en/benchmarks/triviaqa_indic.md create mode 100644 evalscope/docs/en/benchmarks/tvbench.md create mode 100644 evalscope/docs/en/benchmarks/visfactor.md create mode 100644 evalscope/docs/en/benchmarks/vlms_are_biased.md create mode 100644 evalscope/docs/en/benchmarks/vtcbench.md create mode 100644 evalscope/docs/en/benchmarks/wide_search.md create mode 100644 evalscope/docs/en/third_party/deepsearchqa.md create mode 100644 evalscope/docs/en/third_party/miniwob.md create mode 100644 evalscope/docs/en/third_party/skillsbench.md create mode 100644 evalscope/docs/en/third_party/toolathlon.md create mode 100644 evalscope/docs/en/third_party/wide_search.md create mode 100644 evalscope/docs/zh/benchmarks/agieval.md create mode 100644 evalscope/docs/zh/benchmarks/arc_agi_2.md create mode 100644 evalscope/docs/zh/benchmarks/arc_indic.md create mode 100644 evalscope/docs/zh/benchmarks/automation_bench.md create mode 100644 evalscope/docs/zh/benchmarks/bhasha_bench_multi_ayur.md create mode 100644 evalscope/docs/zh/benchmarks/bhasha_bench_multi_finance.md create mode 100644 evalscope/docs/zh/benchmarks/bhasha_bench_multi_krishi.md create mode 100644 evalscope/docs/zh/benchmarks/bhasha_bench_multi_legal.md create mode 100644 evalscope/docs/zh/benchmarks/bhashabenchv1_ayur.md create mode 100644 evalscope/docs/zh/benchmarks/bhashabenchv1_finance.md create mode 100644 evalscope/docs/zh/benchmarks/bhashabenchv1_krishi.md create mode 100644 evalscope/docs/zh/benchmarks/bhashabenchv1_legal.md create mode 100644 evalscope/docs/zh/benchmarks/cc_ocr_v2.md create mode 100644 evalscope/docs/zh/benchmarks/claw_eval.md create mode 100644 evalscope/docs/zh/benchmarks/count_qa.md create mode 100644 evalscope/docs/zh/benchmarks/deep_swe.md create mode 100644 evalscope/docs/zh/benchmarks/deepsearchqa.md create mode 100644 evalscope/docs/zh/benchmarks/emb_spatial_bench.md create mode 100644 evalscope/docs/zh/benchmarks/gsm8k_indic.md create mode 100644 evalscope/docs/zh/benchmarks/hellaswag_hi.md create mode 100644 evalscope/docs/zh/benchmarks/hipho.md create mode 100644 evalscope/docs/zh/benchmarks/hmmt_nov25.md create mode 100644 evalscope/docs/zh/benchmarks/indic_boolq.md create mode 100644 evalscope/docs/zh/benchmarks/indic_param.md create mode 100644 evalscope/docs/zh/benchmarks/job_bench.md create mode 100644 evalscope/docs/zh/benchmarks/kina.md create mode 100644 evalscope/docs/zh/benchmarks/measure_bench.md create mode 100644 evalscope/docs/zh/benchmarks/medxpertqa.md create mode 100644 evalscope/docs/zh/benchmarks/milu.md create mode 100644 evalscope/docs/zh/benchmarks/miniwob.md create mode 100644 evalscope/docs/zh/benchmarks/officeqa.md create mode 100644 evalscope/docs/zh/benchmarks/olmocr_bench.md create mode 100644 evalscope/docs/zh/benchmarks/omni_doc_bench_v1_6.md create mode 100644 evalscope/docs/zh/benchmarks/one_million_bench.md create mode 100644 evalscope/docs/zh/benchmarks/perception_bench.md create mode 100644 evalscope/docs/zh/benchmarks/perspective_gap_prompt_writing.md create mode 100644 evalscope/docs/zh/benchmarks/perspective_gap_role_assignment.md create mode 100644 evalscope/docs/zh/benchmarks/phyx_mc.md create mode 100644 evalscope/docs/zh/benchmarks/phyx_oe.md create mode 100644 evalscope/docs/zh/benchmarks/plawbench.md create mode 100644 evalscope/docs/zh/benchmarks/pmc_vqa.md create mode 100644 evalscope/docs/zh/benchmarks/prbench.md create mode 100644 evalscope/docs/zh/benchmarks/ref_adv_s.md create mode 100644 evalscope/docs/zh/benchmarks/researchrubrics.md create mode 100644 evalscope/docs/zh/benchmarks/sanskriti.md create mode 100644 evalscope/docs/zh/benchmarks/screenspot_pro.md create mode 100644 evalscope/docs/zh/benchmarks/skillsbench.md create mode 100644 evalscope/docs/zh/benchmarks/slake.md create mode 100644 evalscope/docs/zh/benchmarks/surds.md create mode 100644 evalscope/docs/zh/benchmarks/toolathlon.md create mode 100644 evalscope/docs/zh/benchmarks/triviaqa_indic.md create mode 100644 evalscope/docs/zh/benchmarks/tvbench.md create mode 100644 evalscope/docs/zh/benchmarks/visfactor.md create mode 100644 evalscope/docs/zh/benchmarks/vlms_are_biased.md create mode 100644 evalscope/docs/zh/benchmarks/vtcbench.md create mode 100644 evalscope/docs/zh/benchmarks/wide_search.md create mode 100644 evalscope/docs/zh/third_party/deepsearchqa.md create mode 100644 evalscope/docs/zh/third_party/miniwob.md create mode 100644 evalscope/docs/zh/third_party/skillsbench.md create mode 100644 evalscope/docs/zh/third_party/toolathlon.md create mode 100644 evalscope/docs/zh/third_party/wide_search.md create mode 100644 evalscope/evalscope/api/benchmark/adapters/browsergym_adapter.py create mode 100644 evalscope/evalscope/api/judge/__init__.py create mode 100644 evalscope/evalscope/api/judge/aggregation.py create mode 100644 evalscope/evalscope/api/judge/contracts.py create mode 100644 evalscope/evalscope/api/judge/definition.py create mode 100644 evalscope/evalscope/api/judge/executor.py create mode 100644 evalscope/evalscope/api/judge/summary.py create mode 100644 evalscope/evalscope/api/judge/types.py create mode 100644 evalscope/evalscope/api/messages/request_stats.py create mode 100644 evalscope/evalscope/api/metric/semantics.py create mode 100644 evalscope/evalscope/benchmarks/_meta/arc_indic.json create mode 100644 evalscope/evalscope/benchmarks/_meta/automation_bench.json create mode 100644 evalscope/evalscope/benchmarks/_meta/bhasha_bench_multi_ayur.json create mode 100644 evalscope/evalscope/benchmarks/_meta/bhasha_bench_multi_finance.json create mode 100644 evalscope/evalscope/benchmarks/_meta/bhasha_bench_multi_krishi.json create mode 100644 evalscope/evalscope/benchmarks/_meta/bhasha_bench_multi_legal.json create mode 100644 evalscope/evalscope/benchmarks/_meta/bhashabenchv1_ayur.json create mode 100644 evalscope/evalscope/benchmarks/_meta/bhashabenchv1_finance.json create mode 100644 evalscope/evalscope/benchmarks/_meta/bhashabenchv1_krishi.json create mode 100644 evalscope/evalscope/benchmarks/_meta/bhashabenchv1_legal.json create mode 100644 evalscope/evalscope/benchmarks/_meta/cc_ocr_v2.json create mode 100644 evalscope/evalscope/benchmarks/_meta/count_qa.json create mode 100644 evalscope/evalscope/benchmarks/_meta/deepsearchqa.json create mode 100644 evalscope/evalscope/benchmarks/_meta/gsm8k_indic.json create mode 100644 evalscope/evalscope/benchmarks/_meta/hellaswag_hi.json create mode 100644 evalscope/evalscope/benchmarks/_meta/hipho.json create mode 100644 evalscope/evalscope/benchmarks/_meta/hmmt_nov25.json create mode 100644 evalscope/evalscope/benchmarks/_meta/indic_boolq.json create mode 100644 evalscope/evalscope/benchmarks/_meta/indic_param.json create mode 100644 evalscope/evalscope/benchmarks/_meta/job_bench.json create mode 100644 evalscope/evalscope/benchmarks/_meta/medxpertqa.json create mode 100644 evalscope/evalscope/benchmarks/_meta/milu.json create mode 100644 evalscope/evalscope/benchmarks/_meta/miniwob.json create mode 100644 evalscope/evalscope/benchmarks/_meta/olmocr_bench.json create mode 100644 evalscope/evalscope/benchmarks/_meta/omni_doc_bench_v1_6.json create mode 100644 evalscope/evalscope/benchmarks/_meta/one_million_bench.json create mode 100644 evalscope/evalscope/benchmarks/_meta/perception_bench.json create mode 100644 evalscope/evalscope/benchmarks/_meta/phyx_mc.json create mode 100644 evalscope/evalscope/benchmarks/_meta/phyx_oe.json create mode 100644 evalscope/evalscope/benchmarks/_meta/plawbench.json create mode 100644 evalscope/evalscope/benchmarks/_meta/pmc_vqa.json create mode 100644 evalscope/evalscope/benchmarks/_meta/prbench.json create mode 100644 evalscope/evalscope/benchmarks/_meta/ref_adv_s.json create mode 100644 evalscope/evalscope/benchmarks/_meta/sanskriti.json create mode 100644 evalscope/evalscope/benchmarks/_meta/screenspot_pro.json create mode 100644 evalscope/evalscope/benchmarks/_meta/slake.json create mode 100644 evalscope/evalscope/benchmarks/_meta/surds.json create mode 100644 evalscope/evalscope/benchmarks/_meta/triviaqa_indic.json create mode 100644 evalscope/evalscope/benchmarks/_meta/visfactor.json create mode 100644 evalscope/evalscope/benchmarks/_meta/vlms_are_biased.json create mode 100644 evalscope/evalscope/benchmarks/_meta/vtcbench.json create mode 100644 evalscope/evalscope/benchmarks/acebench/checker.py create mode 100644 evalscope/evalscope/benchmarks/acebench/parser.py create mode 100644 evalscope/evalscope/benchmarks/acebench/prompts.py create mode 100644 evalscope/evalscope/benchmarks/acebench/rollout.py create mode 100644 evalscope/evalscope/benchmarks/acebench/scenarios.py create mode 100644 evalscope/evalscope/benchmarks/arc_indic/__init__.py create mode 100644 evalscope/evalscope/benchmarks/arc_indic/arc_indic_adapter.py create mode 100644 evalscope/evalscope/benchmarks/automation_bench/__init__.py create mode 100644 evalscope/evalscope/benchmarks/automation_bench/automation_bench_adapter.py create mode 100644 evalscope/evalscope/benchmarks/automation_bench/utils.py create mode 100644 evalscope/evalscope/benchmarks/bhasha_bench_multi/__init__.py create mode 100644 evalscope/evalscope/benchmarks/bhasha_bench_multi/bhasha_bench_multi_adapter.py create mode 100644 evalscope/evalscope/benchmarks/bhashabenchv1/__init__.py create mode 100644 evalscope/evalscope/benchmarks/bhashabenchv1/bhashabenchv1_adapter.py create mode 100644 evalscope/evalscope/benchmarks/cc_ocr_v2/__init__.py create mode 100644 evalscope/evalscope/benchmarks/cc_ocr_v2/cc_ocr_v2_adapter.py create mode 100644 evalscope/evalscope/benchmarks/cc_ocr_v2/requirements.txt create mode 100644 evalscope/evalscope/benchmarks/cc_ocr_v2/utils.py create mode 100644 evalscope/evalscope/benchmarks/count_qa/__init__.py create mode 100644 evalscope/evalscope/benchmarks/count_qa/count_qa_adapter.py create mode 100644 evalscope/evalscope/benchmarks/deepsearchqa/__init__.py create mode 100644 evalscope/evalscope/benchmarks/deepsearchqa/deepsearchqa_adapter.py create mode 100644 evalscope/evalscope/benchmarks/deepsearchqa/utils.py create mode 100644 evalscope/evalscope/benchmarks/general_qa_vqa_metrics.py create mode 100644 evalscope/evalscope/benchmarks/gsm8k_indic/__init__.py create mode 100644 evalscope/evalscope/benchmarks/gsm8k_indic/gsm8k_indic_adapter.py create mode 100644 evalscope/evalscope/benchmarks/hellaswag_hi/__init__.py create mode 100644 evalscope/evalscope/benchmarks/hellaswag_hi/hellaswag_hi_adapter.py create mode 100644 evalscope/evalscope/benchmarks/hipho/__init__.py create mode 100644 evalscope/evalscope/benchmarks/hipho/hipho_adapter.py create mode 100644 evalscope/evalscope/benchmarks/hipho/utils.py create mode 100644 evalscope/evalscope/benchmarks/hmmt/hmmt_nov25_adapter.py create mode 100644 evalscope/evalscope/benchmarks/indic_boolq/__init__.py create mode 100644 evalscope/evalscope/benchmarks/indic_boolq/indic_boolq_adapter.py create mode 100644 evalscope/evalscope/benchmarks/indic_param/__init__.py create mode 100644 evalscope/evalscope/benchmarks/indic_param/indic_param_adapter.py create mode 100644 evalscope/evalscope/benchmarks/job_bench/__init__.py create mode 100644 evalscope/evalscope/benchmarks/job_bench/job_bench_adapter.py create mode 100644 evalscope/evalscope/benchmarks/job_bench/utils.py create mode 100644 evalscope/evalscope/benchmarks/medxpertqa/__init__.py create mode 100644 evalscope/evalscope/benchmarks/medxpertqa/medxpertqa_adapter.py create mode 100644 evalscope/evalscope/benchmarks/milu/__init__.py create mode 100644 evalscope/evalscope/benchmarks/milu/milu_adapter.py create mode 100644 evalscope/evalscope/benchmarks/miniwob/__init__.py create mode 100644 evalscope/evalscope/benchmarks/miniwob/miniwob_adapter.py create mode 100644 evalscope/evalscope/benchmarks/miniwob/requirements.txt create mode 100644 evalscope/evalscope/benchmarks/miniwob/utils.py create mode 100644 evalscope/evalscope/benchmarks/olmocr_bench/__init__.py create mode 100644 evalscope/evalscope/benchmarks/olmocr_bench/olmocr_bench_adapter.py create mode 100644 evalscope/evalscope/benchmarks/olmocr_bench/requirements.txt create mode 100644 evalscope/evalscope/benchmarks/olmocr_bench/table_parsing.py create mode 100644 evalscope/evalscope/benchmarks/olmocr_bench/unit_tests.py create mode 100644 evalscope/evalscope/benchmarks/omnidoc_bench/legacy/__init__.py rename evalscope/evalscope/benchmarks/omnidoc_bench/{ => legacy}/end2end_eval.py (77%) rename evalscope/evalscope/benchmarks/omnidoc_bench/{ => legacy}/metrics.py (85%) rename evalscope/evalscope/benchmarks/omnidoc_bench/{ => legacy}/omnidoc_bench_adapter.py (62%) rename evalscope/evalscope/benchmarks/omnidoc_bench/{ => legacy}/requirements.txt (100%) rename evalscope/evalscope/benchmarks/omnidoc_bench/{ => legacy}/utils.py (79%) create mode 100644 evalscope/evalscope/benchmarks/omnidoc_bench/v1_6/__init__.py create mode 100644 evalscope/evalscope/benchmarks/omnidoc_bench/v1_6/omnidoc_bench_v1_6_adapter.py create mode 100644 evalscope/evalscope/benchmarks/omnidoc_bench/v1_6/sandbox_scorer.py create mode 100644 evalscope/evalscope/benchmarks/one_million_bench/__init__.py create mode 100644 evalscope/evalscope/benchmarks/one_million_bench/one_million_bench_adapter.py create mode 100644 evalscope/evalscope/benchmarks/perception_bench/__init__.py create mode 100644 evalscope/evalscope/benchmarks/perception_bench/perception_bench_adapter.py create mode 100644 evalscope/evalscope/benchmarks/perception_bench/utils.py create mode 100644 evalscope/evalscope/benchmarks/phyx/__init__.py create mode 100644 evalscope/evalscope/benchmarks/phyx/phyx_adapter.py create mode 100644 evalscope/evalscope/benchmarks/phyx/utils.py create mode 100644 evalscope/evalscope/benchmarks/plawbench/__init__.py create mode 100644 evalscope/evalscope/benchmarks/plawbench/plawbench_adapter.py create mode 100644 evalscope/evalscope/benchmarks/plawbench/utils.py create mode 100644 evalscope/evalscope/benchmarks/pmc_vqa/__init__.py create mode 100644 evalscope/evalscope/benchmarks/pmc_vqa/pmc_vqa_adapter.py create mode 100644 evalscope/evalscope/benchmarks/prbench/__init__.py create mode 100644 evalscope/evalscope/benchmarks/prbench/prbench_adapter.py create mode 100644 evalscope/evalscope/benchmarks/ref_adv_s/__init__.py create mode 100644 evalscope/evalscope/benchmarks/ref_adv_s/ref_adv_s_adapter.py create mode 100644 evalscope/evalscope/benchmarks/ref_adv_s/utils.py create mode 100644 evalscope/evalscope/benchmarks/sanskriti/__init__.py create mode 100644 evalscope/evalscope/benchmarks/sanskriti/sanskriti_adapter.py create mode 100644 evalscope/evalscope/benchmarks/screenspot_pro/__init__.py create mode 100644 evalscope/evalscope/benchmarks/screenspot_pro/screenspot_pro_adapter.py create mode 100644 evalscope/evalscope/benchmarks/screenspot_pro/utils.py create mode 100644 evalscope/evalscope/benchmarks/slake/__init__.py create mode 100644 evalscope/evalscope/benchmarks/slake/slake_adapter.py create mode 100644 evalscope/evalscope/benchmarks/slake/utils.py create mode 100644 evalscope/evalscope/benchmarks/surds/__init__.py create mode 100644 evalscope/evalscope/benchmarks/surds/surds_adapter.py create mode 100644 evalscope/evalscope/benchmarks/surds/utils.py create mode 100644 evalscope/evalscope/benchmarks/triviaqa_indic/__init__.py create mode 100644 evalscope/evalscope/benchmarks/triviaqa_indic/triviaqa_indic_adapter.py create mode 100644 evalscope/evalscope/benchmarks/visfactor/__init__.py create mode 100644 evalscope/evalscope/benchmarks/visfactor/utils.py create mode 100644 evalscope/evalscope/benchmarks/visfactor/visfactor_adapter.py create mode 100644 evalscope/evalscope/benchmarks/vlms_are_biased/__init__.py create mode 100644 evalscope/evalscope/benchmarks/vlms_are_biased/utils.py create mode 100644 evalscope/evalscope/benchmarks/vlms_are_biased/vlms_are_biased_adapter.py create mode 100644 evalscope/evalscope/benchmarks/vtcbench/__init__.py create mode 100644 evalscope/evalscope/benchmarks/vtcbench/requirements.txt create mode 100644 evalscope/evalscope/benchmarks/vtcbench/vtcbench_adapter.py create mode 100644 evalscope/evalscope/evaluation_versioning.py create mode 100644 evalscope/evalscope/evaluator/execution_tracker.py delete mode 100644 evalscope/evalscope/metrics/judge/base.py delete mode 100644 evalscope/evalscope/metrics/judge/score_extractors.py create mode 100644 evalscope/evalscope/metrics/semantics/__init__.py create mode 100644 evalscope/evalscope/metrics/semantics/baselines.py create mode 100644 evalscope/evalscope/metrics/semantics/catalog.py create mode 100644 evalscope/evalscope/metrics/semantics/entry.py create mode 100644 evalscope/evalscope/metrics/semantics/formatting.py create mode 100644 evalscope/evalscope/metrics/semantics/identity.py create mode 100644 evalscope/evalscope/metrics/semantics/legacy.py create mode 100644 evalscope/evalscope/metrics/semantics/migration.py create mode 100644 evalscope/evalscope/metrics/semantics/perf.py create mode 100644 evalscope/evalscope/metrics/semantics/ranking.py create mode 100644 evalscope/evalscope/metrics/semantics/resolver.py create mode 100644 evalscope/evalscope/metrics/semantics/summary.py create mode 100644 evalscope/evalscope/models/utils/async_client.py create mode 100644 evalscope/evalscope/perf/core/pipeline.py create mode 100644 evalscope/evalscope/report/ref.py create mode 100644 evalscope/evalscope/report/template/js/theme_init.js create mode 100644 evalscope/evalscope/service/api_models/__init__.py create mode 100644 evalscope/evalscope/service/api_models/common.py create mode 100644 evalscope/evalscope/service/api_models/eval.py create mode 100644 evalscope/evalscope/service/api_models/perf.py create mode 100644 evalscope/evalscope/service/api_models/reports.py create mode 100644 evalscope/evalscope/service/report_meta_cache.py create mode 100644 evalscope/evalscope/service/responses.py create mode 100644 evalscope/evalscope/third_party/acebench/LICENSE create mode 100644 evalscope/evalscope/third_party/acebench/__init__.py create mode 100644 evalscope/evalscope/third_party/acebench/en/__init__.py create mode 100644 evalscope/evalscope/third_party/acebench/en/base_api.py create mode 100644 evalscope/evalscope/third_party/acebench/en/food_services.py create mode 100644 evalscope/evalscope/third_party/acebench/en/message.py create mode 100644 evalscope/evalscope/third_party/acebench/en/reminder.py create mode 100644 evalscope/evalscope/third_party/acebench/en/travel.py create mode 100644 evalscope/evalscope/third_party/acebench/zh/__init__.py create mode 100644 evalscope/evalscope/third_party/acebench/zh/base_api.py create mode 100644 evalscope/evalscope/third_party/acebench/zh/food_services.py create mode 100644 evalscope/evalscope/third_party/acebench/zh/message.py create mode 100644 evalscope/evalscope/third_party/acebench/zh/reminder.py create mode 100644 evalscope/evalscope/third_party/acebench/zh/travel.py create mode 100644 evalscope/evalscope/utils/asyncio_runtime.py create mode 100644 evalscope/evalscope/utils/download_utils.py create mode 100644 evalscope/evalscope/utils/media_utils.py create mode 100644 evalscope/evalscope/utils/uri_utils.py delete mode 100644 evalscope/evalscope/utils/url_utils.py rename evalscope/evalscope/web/{src/domain => scripts}/drift/localeKeyCheck.test.ts (94%) create mode 100644 evalscope/evalscope/web/scripts/generateApiContracts.ts create mode 100644 evalscope/evalscope/web/src/api/client.test.ts create mode 100644 evalscope/evalscope/web/src/api/generated/contracts.schema.json create mode 100644 evalscope/evalscope/web/src/api/generated/contracts.ts delete mode 100644 evalscope/evalscope/web/src/api/schemas/common.schema.ts delete mode 100644 evalscope/evalscope/web/src/api/schemas/endpointSchemas.test.ts delete mode 100644 evalscope/evalscope/web/src/api/schemas/eval.schema.ts delete mode 100644 evalscope/evalscope/web/src/api/schemas/index.ts delete mode 100644 evalscope/evalscope/web/src/api/schemas/perf.schema.ts delete mode 100644 evalscope/evalscope/web/src/api/schemas/reports.schema.test.ts delete mode 100644 evalscope/evalscope/web/src/api/schemas/reports.schema.ts delete mode 100644 evalscope/evalscope/web/src/api/schemas/schemas.property.test.ts create mode 100644 evalscope/evalscope/web/src/components/charts/MetricTrend.tsx create mode 100644 evalscope/evalscope/web/src/components/chat/AgentTraceView.test.tsx rename evalscope/evalscope/web/src/components/{single => }/chat/AgentTraceView.tsx (62%) create mode 100644 evalscope/evalscope/web/src/components/chat/ChatView.test.tsx rename evalscope/evalscope/web/src/components/{single => chat}/ChatView.tsx (72%) rename evalscope/evalscope/web/src/components/{single => }/chat/EvalResultPanel.test.tsx (53%) rename evalscope/evalscope/web/src/components/{single => }/chat/EvalResultPanel.tsx (75%) create mode 100644 evalscope/evalscope/web/src/components/chat/JudgeReviewPanel.test.tsx create mode 100644 evalscope/evalscope/web/src/components/chat/JudgeReviewPanel.tsx rename evalscope/evalscope/web/src/components/{single => }/chat/MediaBlocks.tsx (80%) rename evalscope/evalscope/web/src/components/{single => }/chat/MessageComponents.tsx (98%) rename evalscope/evalscope/web/src/components/{single => }/chat/ToolCallComponents.tsx (94%) rename evalscope/evalscope/web/src/components/{single => }/chat/roleConfig.ts (76%) create mode 100644 evalscope/evalscope/web/src/components/common/ErrorBoundary.test.tsx delete mode 100644 evalscope/evalscope/web/src/components/common/LoadingSpinner.tsx delete mode 100644 evalscope/evalscope/web/src/components/common/MarkdownRenderer.test.tsx create mode 100644 evalscope/evalscope/web/src/components/compare/CompareReportRail.tsx create mode 100644 evalscope/evalscope/web/src/components/compare/PredictionCompareTab.tsx create mode 100644 evalscope/evalscope/web/src/components/compare/ScoreMatrixTab.tsx create mode 100644 evalscope/evalscope/web/src/components/compare/compareSlots.ts create mode 100644 evalscope/evalscope/web/src/components/dashboard/AggregatedResults.tsx create mode 100644 evalscope/evalscope/web/src/components/dashboard/dashboard.test.tsx create mode 100644 evalscope/evalscope/web/src/components/eval/EvalConfigForm.test.tsx rename evalscope/evalscope/web/src/{pages => components/perf}/PerfRunsTab.tsx (70%) create mode 100644 evalscope/evalscope/web/src/components/reports/SampleNavigator.tsx create mode 100644 evalscope/evalscope/web/src/components/reports/metricCells.test.tsx create mode 100644 evalscope/evalscope/web/src/components/reports/metricCells.tsx delete mode 100644 evalscope/evalscope/web/src/components/single/ChatView.test.tsx rename evalscope/evalscope/web/src/{pages/EvalTaskPage.tsx => components/tasks/EvalTaskPanel.tsx} (60%) rename evalscope/evalscope/web/src/{pages/PerfTaskPage.tsx => components/tasks/PerfTaskPanel.tsx} (66%) create mode 100644 evalscope/evalscope/web/src/components/tasks/TaskFormShell.tsx rename evalscope/evalscope/web/src/components/tasks/{TaskRunnerPage.test.tsx => TaskRunnerShell.test.tsx} (93%) rename evalscope/evalscope/web/src/components/tasks/{TaskRunnerPage.tsx => TaskRunnerShell.tsx} (90%) create mode 100644 evalscope/evalscope/web/src/components/tasks/taskFormContract.test.tsx create mode 100644 evalscope/evalscope/web/src/components/tasks/useTaskForm.ts create mode 100644 evalscope/evalscope/web/src/components/ui/Callout.tsx create mode 100644 evalscope/evalscope/web/src/components/ui/ConfirmDialog.test.tsx create mode 100644 evalscope/evalscope/web/src/components/ui/ConfirmDialog.tsx rename evalscope/evalscope/web/src/components/{common => ui}/ImageLightbox.tsx (100%) rename evalscope/evalscope/web/src/components/{common => ui}/JsonViewer.tsx (100%) delete mode 100644 evalscope/evalscope/web/src/components/ui/KpiCard.tsx create mode 100644 evalscope/evalscope/web/src/components/ui/KpiStrip.tsx create mode 100644 evalscope/evalscope/web/src/components/ui/LabelledField.tsx rename evalscope/evalscope/web/src/components/{common => ui}/LazyCodeBlock.tsx (100%) rename evalscope/evalscope/web/src/components/{common => ui}/LogViewer.tsx (100%) create mode 100644 evalscope/evalscope/web/src/components/ui/MarkdownRenderer.test.tsx rename evalscope/evalscope/web/src/components/{common => ui}/MarkdownRenderer.tsx (96%) create mode 100644 evalscope/evalscope/web/src/components/ui/ScoreBar.test.tsx create mode 100644 evalscope/evalscope/web/src/components/ui/ScoreBar.tsx create mode 100644 evalscope/evalscope/web/src/components/ui/ScoreThresholdInput.tsx create mode 100644 evalscope/evalscope/web/src/components/ui/SegmentedControl.tsx create mode 100644 evalscope/evalscope/web/src/components/ui/Tooltip.test.tsx create mode 100644 evalscope/evalscope/web/src/components/ui/Tooltip.tsx rename evalscope/evalscope/web/src/components/{common => ui}/VirtualList.tsx (100%) rename evalscope/evalscope/web/src/components/{common => ui}/virtualWindow.test.ts (100%) rename evalscope/evalscope/web/src/components/{common => ui}/virtualWindow.ts (100%) create mode 100644 evalscope/evalscope/web/src/contexts/ReportsContext.test.tsx create mode 100644 evalscope/evalscope/web/src/domain/chat/judgeReview.test.ts create mode 100644 evalscope/evalscope/web/src/domain/chat/judgeReview.ts create mode 100644 evalscope/evalscope/web/src/domain/chat/messageText.test.ts rename evalscope/evalscope/web/src/{components/single/chat/chatHelpers.ts => domain/chat/messageText.ts} (70%) create mode 100644 evalscope/evalscope/web/src/domain/compare/scoreMatrix.test.ts create mode 100644 evalscope/evalscope/web/src/domain/compare/scoreMatrix.ts rename evalscope/evalscope/web/src/domain/compare/{compareModel.test.ts => selection.test.ts} (53%) rename evalscope/evalscope/web/src/domain/compare/{compareModel.ts => selection.ts} (51%) delete mode 100644 evalscope/evalscope/web/src/domain/metric/MetricDisplaySpec.ts create mode 100644 evalscope/evalscope/web/src/domain/metric/MetricSemantics.ts create mode 100644 evalscope/evalscope/web/src/domain/metric/__arbitraries__.ts create mode 100644 evalscope/evalscope/web/src/domain/metric/index.ts delete mode 100644 evalscope/evalscope/web/src/domain/metric/registry.test.ts delete mode 100644 evalscope/evalscope/web/src/domain/metric/registry.ts create mode 100644 evalscope/evalscope/web/src/domain/metric/staticConstraints.test.ts rename evalscope/evalscope/web/src/{utils/perf.ts => domain/perf/charts.ts} (58%) delete mode 100644 evalscope/evalscope/web/src/domain/perf/compareModel.test.ts create mode 100644 evalscope/evalscope/web/src/domain/perf/deltaModel.test.ts rename evalscope/evalscope/web/src/domain/perf/{compareModel.ts => deltaModel.ts} (55%) create mode 100644 evalscope/evalscope/web/src/domain/report/nativeScale.test.ts create mode 100644 evalscope/evalscope/web/src/domain/report/primaryMetrics.ts create mode 100644 evalscope/evalscope/web/src/domain/report/reportRef.test.ts create mode 100644 evalscope/evalscope/web/src/domain/report/reportRef.ts create mode 100644 evalscope/evalscope/web/src/domain/report/runAggregation.test.ts create mode 100644 evalscope/evalscope/web/src/domain/report/runAggregation.ts create mode 100644 evalscope/evalscope/web/src/domain/trace/stepGroups.ts create mode 100644 evalscope/evalscope/web/src/domain/trace/textualBlockTrace.test.ts create mode 100644 evalscope/evalscope/web/src/domain/trace/toolLinking.test.ts create mode 100644 evalscope/evalscope/web/src/hooks/useAsyncResource.test.tsx create mode 100644 evalscope/evalscope/web/src/hooks/useAsyncResource.ts create mode 100644 evalscope/evalscope/web/src/hooks/useBatchDelete.test.tsx create mode 100644 evalscope/evalscope/web/src/hooks/useBatchDelete.ts create mode 100644 evalscope/evalscope/web/src/hooks/useScopedState.test.tsx create mode 100644 evalscope/evalscope/web/src/hooks/useScopedState.ts create mode 100644 evalscope/evalscope/web/src/i18n/keyCoverage.test.ts delete mode 100644 evalscope/evalscope/web/src/i18n/translations/performance.ts create mode 100644 evalscope/evalscope/web/src/pages/BenchmarksPage.test.tsx create mode 100644 evalscope/evalscope/web/src/pages/ComparePage.test.tsx create mode 100644 evalscope/evalscope/web/src/pages/ReportDetailPage.test.tsx create mode 100644 evalscope/evalscope/web/src/pages/ReportViewerPage.test.tsx create mode 100644 evalscope/evalscope/web/src/pages/ReportsPage.test.tsx create mode 100644 evalscope/evalscope/web/src/pages/TasksPage.test.tsx create mode 100644 evalscope/evalscope/web/src/pages/dashboardFilter.test.tsx create mode 100644 evalscope/evalscope/web/src/utils/media.test.ts create mode 100644 evalscope/evalscope/web/src/utils/media.ts delete mode 100644 evalscope/evalscope/web/src/utils/reportParser.ts create mode 100644 evalscope/evalscope/web/tsconfig.test.json create mode 100644 evalscope/scripts/generate_web_api_contracts.py create mode 100644 evalscope/scripts/release/verify_package.py create mode 100644 evalscope/tests/agent/external/test_anthropic_bridge_prompt_cache.py create mode 100644 evalscope/tests/agent/external/test_bridge_lifecycle.py create mode 100644 evalscope/tests/agent/external/test_runner_skill_install.py create mode 100644 evalscope/tests/agent/test_docker_image.py create mode 100644 evalscope/tests/agent/test_native_runner.py create mode 100644 evalscope/tests/agent/test_skill_runtime.py create mode 100644 evalscope/tests/api/judge/test_aggregation_excludes_unusable.py create mode 100644 evalscope/tests/api/judge/test_contracts.py create mode 100644 evalscope/tests/api/judge/test_executor.py create mode 100644 evalscope/tests/api/judge/test_gates.py create mode 100644 evalscope/tests/api/judge/test_judge_config.py create mode 100644 evalscope/tests/api/judge/test_llm_judge_mixin.py create mode 100644 evalscope/tests/api/judge/test_llm_recall.py create mode 100644 evalscope/tests/api/judge/test_migrated_adapters.py create mode 100644 evalscope/tests/api/test_benchmark_meta.py create mode 100644 evalscope/tests/api/test_dataset_builder.py create mode 100644 evalscope/tests/api/test_dataset_dict_limit.py create mode 100644 evalscope/tests/api/test_dataset_hub.py create mode 100644 evalscope/tests/api/test_dataset_loader.py create mode 100644 evalscope/tests/api/test_default_data_adapter.py create mode 100644 evalscope/tests/api/test_evaluation_versioning.py create mode 100644 evalscope/tests/api/test_io_utils.py create mode 100644 evalscope/tests/api/test_messages_to_markdown.py create mode 100644 evalscope/tests/api/test_request_stats.py create mode 100644 evalscope/tests/api/test_scoring_policy.py create mode 100644 evalscope/tests/api/test_shuffle_determinism.py create mode 100644 evalscope/tests/api/test_task_config_validation.py create mode 100644 evalscope/tests/api/test_vision_language_adapter.py create mode 100644 evalscope/tests/benchmark/test_acebench.py delete mode 100644 evalscope/tests/benchmark/test_asr_filters.py create mode 100644 evalscope/tests/benchmark/test_bbh_prompts.py create mode 100644 evalscope/tests/benchmark/test_bfcl_v4.py create mode 100644 evalscope/tests/benchmark/test_cc_ocr_v2.py create mode 100644 evalscope/tests/benchmark/test_cmmmu_adapter.py create mode 100644 evalscope/tests/benchmark/test_code_execution_timeouts.py create mode 100644 evalscope/tests/benchmark/test_count_qa.py create mode 100644 evalscope/tests/benchmark/test_deep_swe.py create mode 100644 evalscope/tests/benchmark/test_deepsearchqa.py create mode 100644 evalscope/tests/benchmark/test_general_qa_adapter.py create mode 100644 evalscope/tests/benchmark/test_general_vmcq_adapter.py create mode 100644 evalscope/tests/benchmark/test_general_vqa_adapter.py create mode 100644 evalscope/tests/benchmark/test_gpqa_determinism.py create mode 100644 evalscope/tests/benchmark/test_ifbench_instructions.py create mode 100644 evalscope/tests/benchmark/test_ifeval_language_checkers.py create mode 100644 evalscope/tests/benchmark/test_judge_output_parsing.py create mode 100644 evalscope/tests/benchmark/test_live_code_bench_prompt.py create mode 100644 evalscope/tests/benchmark/test_logic_vista.py create mode 100644 evalscope/tests/benchmark/test_miniwob.py create mode 100644 evalscope/tests/benchmark/test_olmocr_bench.py create mode 100644 evalscope/tests/benchmark/test_omnidoc_bench.py create mode 100644 evalscope/tests/benchmark/test_one_million_bench.py create mode 100644 evalscope/tests/benchmark/test_phyx.py create mode 100644 evalscope/tests/benchmark/test_prbench.py create mode 100644 evalscope/tests/benchmark/test_process_bench.py create mode 100644 evalscope/tests/benchmark/test_ref_adv_s.py create mode 100644 evalscope/tests/benchmark/test_researchrubrics.py create mode 100644 evalscope/tests/benchmark/test_screenspot_pro.py create mode 100644 evalscope/tests/benchmark/test_skillsbench.py create mode 100644 evalscope/tests/benchmark/test_slake.py create mode 100644 evalscope/tests/benchmark/test_surds.py create mode 100644 evalscope/tests/benchmark/test_terminal_bench.py create mode 100644 evalscope/tests/benchmark/test_toolathlon_client.py create mode 100644 evalscope/tests/benchmark/test_toolathlon_remote.py create mode 100644 evalscope/tests/benchmark/test_visfactor.py create mode 100644 evalscope/tests/benchmark/test_vlms_are_biased.py create mode 100644 evalscope/tests/benchmark/test_vtcbench.py create mode 100644 evalscope/tests/benchmark/test_wide_search.py create mode 100644 evalscope/tests/cli/test_metric_semantics_e2e.py create mode 100644 evalscope/tests/cli/test_service.py create mode 100644 evalscope/tests/evaluator/test_cache_manager.py create mode 100644 evalscope/tests/evaluator/test_execution_completeness.py create mode 100644 evalscope/tests/metrics/aggregators/test_missing_metric_keys.py create mode 100644 evalscope/tests/metrics/aggregators/test_weighted_mean.py create mode 100644 evalscope/tests/metrics/test_nlp_metrics.py create mode 100644 evalscope/tests/metrics/test_rouge.py create mode 100644 evalscope/tests/models/test_anthropic_prompt_cache.py create mode 100644 evalscope/tests/models/test_anthropic_stream_retry.py create mode 100644 evalscope/tests/models/test_anthropic_tool_id_sanitize.py create mode 100644 evalscope/tests/models/test_async_client_lifecycle.py create mode 100644 evalscope/tests/models/test_empty_completion_retry.py create mode 100644 evalscope/tests/models/test_generation_config_extra_validation.py create mode 100644 evalscope/tests/models/test_image_model_output.py create mode 100644 evalscope/tests/models/test_litellm_reasoning_history.py create mode 100644 evalscope/tests/models/test_model_cache.py create mode 100644 evalscope/tests/models/test_modelscope_usage.py create mode 100644 evalscope/tests/models/test_no_retry_exceptions.py create mode 100644 evalscope/tests/models/test_openai_audio_input.py create mode 100644 evalscope/tests/models/test_openai_stream_finish_reason.py create mode 100644 evalscope/tests/models/test_openai_stream_reasoning.py create mode 100644 evalscope/tests/models/test_openai_stream_retry.py create mode 100644 evalscope/tests/perf/test_arguments_secrets.py create mode 100644 evalscope/tests/perf/test_arguments_validation.py create mode 100644 evalscope/tests/perf/test_async_lifecycle.py create mode 100644 evalscope/tests/perf/test_chat_template_error.py create mode 100644 evalscope/tests/perf/test_closed_loop_warmup_handoff.py create mode 100644 evalscope/tests/perf/test_dataset_args.py create mode 100644 evalscope/tests/perf/test_line_by_line_body.py create mode 100644 evalscope/tests/perf/test_metrics_consumer.py create mode 100644 evalscope/tests/perf/test_multi_turn_warmup_handoff.py create mode 100644 evalscope/tests/perf/test_open_loop_strategy.py create mode 100644 evalscope/tests/perf/test_percentile_metrics.py create mode 100644 evalscope/tests/perf/test_perf_archive.py create mode 100644 evalscope/tests/perf/test_perf_report_loader.py create mode 100644 evalscope/tests/perf/test_prefix_injection.py create mode 100644 evalscope/tests/perf/test_request_build_errors.py create mode 100644 evalscope/tests/perf/test_stream_metrics.py create mode 100644 evalscope/tests/perf/test_target_input_len.py create mode 100644 evalscope/tests/perf/test_wall_time_failures.py create mode 100644 evalscope/tests/perf/test_workload_trace.py create mode 100644 evalscope/tests/perf/test_workload_trace_e2e.py create mode 100644 evalscope/tests/rag/test_backend_manager.py create mode 100644 evalscope/tests/report/semantics/golden_samples.json create mode 100644 evalscope/tests/report/semantics/test_baselines.py create mode 100644 evalscope/tests/report/semantics/test_benchmark_legacy_rules.py create mode 100644 evalscope/tests/report/semantics/test_builtin_report_contracts.py create mode 100644 evalscope/tests/report/semantics/test_catalog.py create mode 100644 evalscope/tests/report/semantics/test_complex_aggregations.py create mode 100644 evalscope/tests/report/semantics/test_formatting.py create mode 100644 evalscope/tests/report/semantics/test_golden_samples.py create mode 100644 evalscope/tests/report/semantics/test_naming.py create mode 100644 evalscope/tests/report/semantics/test_perf_semantics.py create mode 100644 evalscope/tests/report/semantics/test_public_api.py create mode 100644 evalscope/tests/report/semantics/test_ranking.py create mode 100644 evalscope/tests/report/semantics/test_registry_alignment.py create mode 100644 evalscope/tests/report/semantics/test_report_model.py create mode 100644 evalscope/tests/report/semantics/test_resolver.py create mode 100644 evalscope/tests/report/semantics/test_semantics_model.py create mode 100644 evalscope/tests/report/semantics/test_structured_aggregators.py create mode 100644 evalscope/tests/report/test_execution_summary.py create mode 100644 evalscope/tests/report/test_prediction_media_paths.py create mode 100644 evalscope/tests/report/test_renderer_semantics.py create mode 100644 evalscope/tests/report/test_report_delete.py create mode 100644 evalscope/tests/report/test_report_endpoints.py create mode 100644 evalscope/tests/report/test_report_meta_cache.py create mode 100644 evalscope/tests/report/test_report_ref.py create mode 100644 evalscope/tests/report/test_service_charts.py create mode 100644 evalscope/tests/report/test_static_report_theme.py create mode 100644 evalscope/tests/service/__init__.py create mode 100644 evalscope/tests/service/test_api_contracts.py create mode 100644 evalscope/tests/service/test_eval_semantic_table.py create mode 100644 evalscope/tests/service/test_process_stop.py delete mode 100644 evalscope/tests/test_async_generate.py create mode 100644 evalscope/tests/test_download_utils.py create mode 100644 evalscope/tests/test_function_utils.py delete mode 100644 evalscope/tests/test_loader_shuffle.py create mode 100644 evalscope/tests/test_logger.py create mode 100644 evalscope/tests/test_multi_choices.py delete mode 100644 evalscope/tests/test_openai_responses.py create mode 100644 evalscope/tests/test_resource_utils.py create mode 100644 patches/local-uncommitted-evalscope.patch create mode 100644 patches/local/evalscope/api/evaluator/cache.py.patch create mode 100644 patches/local/evalscope/api/messages/__init__.py.patch create mode 100644 patches/local/evalscope/api/model/generate_config.py.patch create mode 100644 patches/local/evalscope/api/model/model.py.patch create mode 100644 patches/local/evalscope/benchmarks/deep_swe/deep_swe_adapter.py.patch create mode 100644 patches/local/evalscope/benchmarks/swe_bench/utils.py.patch create mode 100644 patches/local/evalscope/benchmarks/terminal_bench/terminal_bench_adapter.py.patch create mode 100644 patches/local/evalscope/evaluator/evaluator.py.patch create mode 100644 patches/local/evalscope/models/anthropic_compatible.py.patch create mode 100644 patches/local/evalscope/models/litellm_compatible.py.patch create mode 100644 patches/local/evalscope/models/openai_compatible.py.patch create mode 100644 patches/local/evalscope/models/openai_responses.py.patch create mode 100644 patches/local/evalscope/models/utils/openai.py.patch create mode 100644 patches/local/evalscope/report/combinator.py.patch create mode 100644 patches/local/evalscope/utils/function_utils.py.patch create mode 100644 patches/local/request_stats.py create mode 100644 patches/local/tests/test_request_stats.py create mode 100644 patches/upstream-base.txt create mode 100644 patches/upstream.sha diff --git a/bash/collect_results.py b/bash/collect_results.py index 14ff756..6ad69cc 100644 --- a/bash/collect_results.py +++ b/bash/collect_results.py @@ -16,6 +16,7 @@ import argparse import json import statistics from pathlib import Path +from typing import Optional import numpy as np import pandas as pd @@ -322,15 +323,78 @@ def parse_log_duration(log_file: Path): return (last_dt - first_dt).total_seconds() / 3600.0 +def _metric_name(metric: dict) -> Optional[str]: + """Return a metric's display/key name across report schema v1 and v2.""" + if not isinstance(metric, dict): + return None + if metric.get('name'): + return str(metric['name']) + identity = metric.get('identity') or {} + if isinstance(identity, dict) and identity.get('name'): + return str(identity['name']) + if metric.get('legacy_name'): + return str(metric['legacy_name']) + return None + + +def _identity_key(identity: Optional[dict]) -> Optional[tuple]: + if not isinstance(identity, dict) or not identity.get('name'): + return None + dims = identity.get('dimensions') or {} + if not isinstance(dims, dict): + dims = {} + return ( + str(identity.get('name')), + str(identity.get('aggregation') or 'mean'), + tuple(sorted((str(k), str(v)) for k, v in dims.items())), + ) + + def extract_score(report_data: dict) -> float: - """Extract the top-level score from a report JSON.""" + """Extract the primary score from a report JSON. + + Supports: + - legacy reports with top-level ``score`` / metrics named ``mean_acc`` + - EvalScope report schema v2 with ``primary_metric_identity`` + + ``metrics[].identity`` / ``metrics[].score`` + """ score = report_data.get('score') if score is not None: return float(score) - metrics = report_data.get('metrics', []) + + metrics = report_data.get('metrics') or [] + if not metrics: + return 0.0 + + # Schema v2: prefer the explicit primary metric identity when present. + primary_identity = report_data.get('primary_metric_identity') + primary_key = _identity_key(primary_identity) + if primary_key is not None: + for m in metrics: + if _identity_key(m.get('identity')) == primary_key: + return float(m.get('score', m.get('macro_score', 0.0))) + + # Legacy / fallback preferred names. + preferred = { + 'mean_acc', + 'accuracy', + 'acc', + 'main_problem_pass_rate', + 'pass_rate', + 'normalized_score', + 'f1', + } for m in metrics: - if m.get('name') == 'mean_acc': + name = _metric_name(m) + if name in preferred: return float(m.get('score', m.get('macro_score', 0.0))) + + # Last resort: first metric with a numeric score. + for m in metrics: + if m.get('score') is not None: + return float(m.get('score')) + if m.get('macro_score') is not None: + return float(m.get('macro_score')) return 0.0 @@ -343,6 +407,9 @@ def collect_benchmark(output_dir: Path, benchmark: str, model_name: str): scores = [] summary0 = None n_samples_unique = 0 + req_success = 0 + req_failed = 0 + req_client = 0 for report in reports: try: data = json.loads(report.read_text(encoding='utf-8')) @@ -351,6 +418,10 @@ def collect_benchmark(output_dir: Path, benchmark: str, model_name: str): perf_metrics = data.get('perf_metrics') or {} summary0 = perf_metrics.get('summary', {}) n_samples_unique = summary0.get('n_samples', data.get('num', 0)) + req = ((data.get('perf_metrics') or {}).get('summary') or {}).get('request') or {} + req_success += int(req.get('success_attempts') or 0) + req_failed += int(req.get('failed_attempts') or 0) + req_client += int(req.get('client_errors') or 0) except Exception: continue @@ -579,12 +650,19 @@ def collect_benchmark(output_dir: Path, benchmark: str, model_name: str): else: duration_hours = max(duration_hours, compute_wall_estimate) + vendor_http = req_success + req_failed + request_success_rate = (req_success / vendor_http) if vendor_http > 0 else np.nan + return { '分类': BENCHMARK_DOMAIN.get(benchmark, '其他'), 'Benchmark': BENCHMARK_NAME_ALIAS.get(benchmark, benchmark), '得分': round(avg_score, 4), '实测时间(h)': round(duration_hours, 4) if not np.isnan(duration_hours) else np.nan, '总样本数': n_samples_unique, + '请求成功率': round(request_success_rate, 4) if not np.isnan(request_success_rate) else np.nan, + 'HTTP成功': req_success if vendor_http or req_client else np.nan, + 'HTTP失败': req_failed if vendor_http or req_client else np.nan, + 'client_errors': req_client if vendor_http or req_client else np.nan, '延迟_mean(s)': round(latency_mean, 5) if not np.isnan(latency_mean) else np.nan, '输出TPS': round(avg_output_tps, 2) if not np.isnan(avg_output_tps) else np.nan, '请求QPS': round(avg_req_ps, 4) if not np.isnan(avg_req_ps) else np.nan, diff --git a/bash/run.py b/bash/run.py index c4ea459..7b42ed6 100644 --- a/bash/run.py +++ b/bash/run.py @@ -127,7 +127,30 @@ ALL_SINGLE_RUN = [ 'openai_mrcr', 'longbench_v2', ] ALL_AGENT = ['tau2_bench', 'general_fc'] -K3_SINGLE = ["gpqa_diamond", "hle", "terminal_bench_v2", "browsecomp", "mcp_atlas", "officeqa", "deepsearchqa", "jobbench", "automation_bench"] +# Kimi-K3 可直接跑且备注为空的集 + HLE-Full（dataset id 为 hle）。 +# tau3 子集 banking_knowledge 在 config/kimi-k3.yaml 里配置。 +K3_SINGLE = [ + # Coding + 'deep_swe', + 'terminal_bench_v2_1', + 'scicode', + # Agentic + 'browsecomp', + 'deepsearchqa', + 'job_bench', + 'officeqa', + 'tau3_bench', + 'researchrubrics', + # Reasoning + 'gpqa_diamond', + 'aa_lcr', + 'hle', + # Vision + 'mmmu_pro', + 'charxiv', + 'math_vision', + 'omni_doc_bench', +] # ============================================================ # Fingerprint / model-identity benchmarks @@ -558,6 +581,22 @@ def build_task_config( else: work_dir = Path(output_dir) / dataset_name / f'seed_{seed}' work_dir.mkdir(parents=True, exist_ok=True) + # 升级后的 EvalScope：只要设置了 use_cache，就会校验 evaluation identity。 + # 空目录（无 task_config.yaml）会被当成 previous=unknown 直接报错； + # 仅有 task_config、无 predictions 的失败/改配残留也会因 fingerprint 变化秒挂。 + # 仅在已有完整缓存快照（config + 至少一个 prediction）时才启用 resume。 + cache_snapshot = work_dir / 'configs' / 'task_config.yaml' + pred_dir = work_dir / 'predictions' + has_predictions = pred_dir.is_dir() and any(pred_dir.rglob('*')) + if cache_snapshot.is_file() and has_predictions: + use_cache = str(work_dir) + else: + if cache_snapshot.is_file() and not has_predictions: + print( + f' [cache] skip resume for {dataset_name}: ' + 'stale/incomplete cache (task_config without predictions)' + ) + use_cache = None work_dir = str(work_dir) generation_config = configure_thinking( @@ -599,7 +638,7 @@ def build_task_config( collect_perf=True, no_timestamp=True, work_dir=work_dir, - use_cache=work_dir, + use_cache=use_cache, datasets=[dataset_name], generation_config=generation_config, dataset_args=dataset_args_dict, diff --git a/config/dpv4-int8_nothinking.yaml b/config/dpv4-int8_nothinking.yaml index 5732d91..0480c73 100644 --- a/config/dpv4-int8_nothinking.yaml +++ b/config/dpv4-int8_nothinking.yaml @@ -203,16 +203,16 @@ swe_bench_pro: top_p: 1.0 stream: true max_tokens: 32768 -terminal_bench_v2_1: - generation_config: - temperature: 0.0 - top_p: 1.0 - stream: true - max_tokens: 8192 - dataset_args: - extra_params: - timeout_multiplier: 2.0 - max_turns: 500 +terminal_bench_v2_1: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 + dataset_args: + extra_params: + timeout_multiplier: 2.0 + max_turns: 500 aa_lcr: generation_config: temperature: 1.0 @@ -269,4 +269,117 @@ toolathlon: temperature: 1.0 top_p: 1.0 stream: true - max_tokens: 32768 \ No newline at end of file + max_tokens: 32768 + dataset_args: + extra_params: + server_host: 127.0.0.1 + server_port: 8080 + ws_proxy_port: 8081 + timeout_seconds: 14400 + submit_timeout_seconds: 300 + task_list: + - paper-checker + skip_container_restart: false + override_output_dir: true +browsecomp: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +mcp_atlas: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +officeqa: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + dataset_args: + subset_list: + - officeqa_pro +deepsearchqa: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +job_bench: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +automation_bench: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + dataset_args: + extra_params: + toolset: api +mmmu_pro: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + dataset_args: + max_image_bytes: 5mb + extra_params: + dataset_format: standard (4 options) +charxiv: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + dataset_args: + subset_list: + - reasoning +math_vision: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +baby_vision: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +zerobench: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + dataset_args: + max_image_bytes: 10mb +world_vqa: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +omni_doc_bench: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + dataset_args: + extra_params: + match_method: quick_match +perception_bench: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 diff --git a/config/dpv4-int8_thinking.yaml b/config/dpv4-int8_thinking.yaml index 2841c21..2f37153 100644 --- a/config/dpv4-int8_thinking.yaml +++ b/config/dpv4-int8_thinking.yaml @@ -299,3 +299,141 @@ toolathlon: stream: true max_tokens: 32768 max_completion_tokens: 64000 + dataset_args: + extra_params: + server_host: 127.0.0.1 + server_port: 8080 + ws_proxy_port: 8081 + timeout_seconds: 14400 + submit_timeout_seconds: 300 + task_list: + - paper-checker + skip_container_restart: false + override_output_dir: true +terminal_bench_v2_1: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 + dataset_args: + extra_params: + timeout_multiplier: 2.0 + max_turns: 500 +browsecomp: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 +mcp_atlas: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 +officeqa: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 + dataset_args: + subset_list: + - officeqa_pro +deepsearchqa: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 +job_bench: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 +automation_bench: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 + dataset_args: + extra_params: + toolset: api +mmmu_pro: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 + dataset_args: + max_image_bytes: 5mb + extra_params: + dataset_format: standard (4 options) +charxiv: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 + dataset_args: + subset_list: + - reasoning +math_vision: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 +baby_vision: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 +zerobench: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 + dataset_args: + max_image_bytes: 10mb +world_vqa: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 +omni_doc_bench: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 + dataset_args: + extra_params: + match_method: quick_match +perception_bench: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 diff --git a/evalscope/.github/workflows/ci_test_full.yaml b/evalscope/.github/workflows/ci_test_full.yaml index 525f6e4..c985848 100644 --- a/evalscope/.github/workflows/ci_test_full.yaml +++ b/evalscope/.github/workflows/ci_test_full.yaml @@ -32,8 +32,10 @@ jobs: run: | python -m pip install --upgrade pip pip install -e '.[dev,ifeval,ifbench,multi_if,needle_haystack,arena_hard]' + pip install litellm==1.95.0 pip install git+https://github.com/sierra-research/tau-bench pip install bfcl-eval==2025.10.27.1 + python -c "import tau_bench" - name: Create .env file run: | diff --git a/evalscope/.github/workflows/code-review.yaml b/evalscope/.github/workflows/code-review.yaml new file mode 100644 index 0000000..2c1e327 --- /dev/null +++ b/evalscope/.github/workflows/code-review.yaml @@ -0,0 +1,39 @@ +name: Qoder Auto Code Review + +# Manually triggered: add the "qoder-review" label to a PR. This avoids running +# on every commit (the previous `synchronize` trigger) and lets a maintainer +# decide when a review is needed. Because the trigger is pull_request_target, +# the job is natively associated with the PR head and shows up as a PR check. +on: + pull_request_target: + types: [labeled] + +jobs: + qoder-review: + # Only run when the label added is exactly "qoder-review". Adding a label + # requires write/triage access, so the labeling action is itself the human + # gate that authorizes running fork PR code in this trusted context. + if: github.event.label.name == 'qoder-review' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + id-token: write + + steps: + - name: Checkout PR head + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + # Fork code checkout is opted in; the qoder-review label (settable only + # by trusted users) is the gate that makes this safe. + allow-unsafe-pr-checkout: true + + - name: Run Qoder Code Review + uses: QoderAI/qoder-action@v0 + with: + qoder_personal_access_token: ${{ secrets.QODER_PERSONAL_ACCESS_TOKEN }} + prompt: | + /review-pr + REPO:${{ github.repository }} PR_NUMBER:${{ github.event.pull_request.number }} diff --git a/evalscope/.github/workflows/frontend.yml b/evalscope/.github/workflows/frontend.yml new file mode 100644 index 0000000..9d16b7a --- /dev/null +++ b/evalscope/.github/workflows/frontend.yml @@ -0,0 +1,71 @@ +name: Frontend checks + +on: + pull_request: + paths: + - 'evalscope/web/**' + - 'evalscope/service/**' + - 'evalscope/report/report.py' + - 'evalscope/api/agent/trace.py' + - 'evalscope/api/metric/**' + - 'evalscope/constants.py' + - 'scripts/generate_web_api_contracts.py' + - 'tests/service/test_api_contracts.py' + - 'tests/report/test_report_endpoints.py' + - 'requirements/dev.txt' + - 'requirements/service.txt' + - 'pyproject.toml' + - 'DESIGN.md' + - '.github/workflows/frontend.yml' + push: + branches: + - main + paths: + - 'evalscope/web/**' + - 'evalscope/service/**' + - 'evalscope/report/report.py' + - 'evalscope/api/agent/trace.py' + - 'evalscope/api/metric/**' + - 'evalscope/constants.py' + - 'scripts/generate_web_api_contracts.py' + - 'tests/service/test_api_contracts.py' + - 'tests/report/test_report_endpoints.py' + - 'requirements/dev.txt' + - 'requirements/service.txt' + - 'pyproject.toml' + - 'DESIGN.md' + - '.github/workflows/frontend.yml' + +jobs: + frontend: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: pip + + - name: Install Python contract dependencies + run: pip install -e '.[dev,service]' + + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: npm + cache-dependency-path: evalscope/web/package-lock.json + + - run: npm ci + working-directory: evalscope/web + - run: npm run contracts:check + working-directory: evalscope/web + - run: python -m pytest tests/service/test_api_contracts.py tests/report/test_report_endpoints.py + - run: npm run lint + working-directory: evalscope/web + - run: npm test + working-directory: evalscope/web + - run: npm run drift + working-directory: evalscope/web + - run: npm run build + working-directory: evalscope/web diff --git a/evalscope/.github/workflows/lint.yml b/evalscope/.github/workflows/lint.yml index 305e1ae..3ae097d 100644 --- a/evalscope/.github/workflows/lint.yml +++ b/evalscope/.github/workflows/lint.yml @@ -26,7 +26,7 @@ jobs: - name: Install pre-commit run: | python -m pip install --upgrade pip - pip install pre-commit + pip install 'pre-commit==4.6.0' - name: Run pre-commit - run: pre-commit run --all-files + run: pre-commit run --all-files --show-diff-on-failure diff --git a/evalscope/.github/workflows/publish.yaml b/evalscope/.github/workflows/publish.yaml index 437b76b..4310849 100644 --- a/evalscope/.github/workflows/publish.yaml +++ b/evalscope/.github/workflows/publish.yaml @@ -25,13 +25,9 @@ jobs: uses: actions/setup-node@v4 with: node-version: '22' - - name: Build frontend - run: make web-build - - name: Install wheel - run: pip install wheel && pip install -r requirements/framework.txt - - name: Build EvalScope - run: python setup.py sdist bdist_wheel + - name: Install build dependencies + run: pip install build twine '.[service]' + - name: Build and verify EvalScope + run: make package - name: Publish package to PyPI - run: | - pip install twine - twine upload dist/* --skip-existing -u __token__ -p ${{ secrets.PYPI_TOKEN }} + run: twine upload dist/* --skip-existing -u __token__ -p ${{ secrets.PYPI_TOKEN }} diff --git a/evalscope/.gitignore b/evalscope/.gitignore index f8e514d..3923746 100644 --- a/evalscope/.gitignore +++ b/evalscope/.gitignore @@ -133,6 +133,7 @@ result.mp4 # personal info private/ +outreach/ # others *.tokenization @@ -165,6 +166,9 @@ docs/zh/learn/ examples/api_test/ .qoder .claude +.kiro +.pr-review/ +.scratch_wl/ # Frontend (evalscope/web) node_modules/ @@ -172,13 +176,6 @@ evalscope/web/dist/ evalscope/web/.vite/ ragas_mini_testset_score.json -# User benchmark outputs and secrets -benchmark_outputs/ -test_scripts/ -*.secret -*.key -api_keys.json - # Docker build artifacts (pre-cloned agent sources) evalscope/agent/external/dockerfiles/hermes-agent-src/ evalscope/agent/external/dockerfiles/hermes-install.sh diff --git a/evalscope/.pre-commit-config.yaml b/evalscope/.pre-commit-config.yaml index e650b66..03e3008 100644 --- a/evalscope/.pre-commit-config.yaml +++ b/evalscope/.pre-commit-config.yaml @@ -1,52 +1,23 @@ repos: - - repo: https://github.com/pycqa/flake8.git - rev: 7.3.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.16.4 hooks: - - id: flake8 - exclude: | - (?x)^( - examples/| - docs/| - tests/| - evalscope/utils/utils.py| - evalscope/third_party/| - evalscope/backend/rag_eval/clip_benchmark/tasks| - evalscope/backend/rag_eval/cmteb/tasks| - evalscope/metrics/vision/t2v_metrics - ) - - repo: https://github.com/PyCQA/isort.git - rev: 7.0.0 - hooks: - - id: isort - - repo: https://github.com/google/yapf - rev: v0.43.0 - hooks: - - id: yapf - exclude: | - (?x)^( - examples/| - docs/| - tests/| - evalscope/utils/utils.py| - evalscope/third_party/| - evalscope/backend/rag_eval/clip_benchmark/tasks| - evalscope/backend/rag_eval/cmteb/tasks - ) + - id: ruff-check + args: [--fix] + - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks.git rev: v6.0.0 hooks: - id: trailing-whitespace - exclude: thirdparty/|docs/|examples + exclude: evalscope/third_party/|docs/|examples - id: check-yaml - exclude: thirdparty/|docs/|examples + exclude: evalscope/third_party/|docs/|examples - id: end-of-file-fixer - exclude: thirdparty/|docs/|examples|.*\.json + exclude: evalscope/third_party/|docs/|examples|.*\.json - id: requirements-txt-fixer - exclude: thirdparty/|docs/|examples - - id: double-quote-string-fixer - exclude: thirdparty/|docs/|examples|.*\.json|cl_bench_adapter.py + exclude: evalscope/third_party/|docs/|examples - id: check-merge-conflict - exclude: thirdparty/|docs/|examples + exclude: evalscope/third_party/|docs/|examples - id: mixed-line-ending - exclude: thirdparty/|docs/|examples + exclude: evalscope/third_party/|docs/|examples args: ["--fix=lf"] diff --git a/evalscope/AGENTS.md b/evalscope/AGENTS.md index 77f784d..401e6fa 100644 --- a/evalscope/AGENTS.md +++ b/evalscope/AGENTS.md @@ -14,7 +14,7 @@ Python ≥ 3.10 (3.10 / 3.11 / 3.12). Dependencies: `requirements/framework.txt` ## Build, lint, test ```bash -make lint # required before commit (yapf + isort + flake8 + basic pre-commit hooks) +make lint # apply Ruff fixes/formatting and run all pre-commit checks pytest tests/cli/test_all.py::TestRun::test_ci_lite -v -s -p no:warnings # CI smoke test pytest tests/perf/test_perf_basic.py::TestPerfBasic::test_multi_parallel_sweep -v -s # perf ``` @@ -25,6 +25,15 @@ Commits failing `make lint` are rejected on `main`. Benchmark detail pages (`docs/{zh,en}/benchmarks/.md`) and meta cache (`evalscope/benchmarks/_meta/.json`) are **auto-generated** from each adapter's `BenchmarkMeta.description` + dataset statistics. Do not hand-edit those files. +Every `BenchmarkMeta.description` must be English Markdown with these sections in this order: + +1. `## Overview`: benchmark purpose and scope. +2. `## Task Description`: bullet fields for `Task Type`, `Input`, `Output`, and `Domain` (use a more precise fourth field such as `Modalities` or `Grading` only when `Domain` does not apply). +3. `## Key Features`: dataset scale/source, evaluated capabilities, and version-specific behavior. +4. `## Evaluation Notes`: metrics, scoring procedure, runtime/dependency requirements, and compatibility limits. + +Do not replace these required headings with benchmark-specific headings. Add extra sections only when the four required sections are insufficient. + When you add a benchmark or change its `BenchmarkMeta.description`, run: ```bash @@ -54,9 +63,10 @@ run_task(TaskConfig(model='Qwen/Qwen2.5-0.5B-Instruct', datasets=['gsm8k'], limi ## Code style (enforced) - **Line width 120**, 4-space indent, LF endings, trailing newline at EOF. -- **Quotes** governed by `double-quote-string-fixer` hook — follow existing file style; do not mix. +- **Quotes**: single quotes, enforced by the Ruff formatter. +- **Linting**: Ruff's `E`, `F`, and `W` rules for maintained source files. +- **Imports**: Ruff's `I` rules, with `evalscope` detected as first-party and standard import sections. - **f-strings** for formatting (no `%` or `.format()` unless necessary). -- **Imports**: isort with `first_party = evalscope`, groups `STDLIB / THIRDPARTY / LOCALFOLDER`, `multi_line_output=3`. - **Type hints required** on every function signature. - **English only** for comments and docstrings. - **Public APIs need docstrings**; internal helpers only when intent is non-obvious. @@ -71,13 +81,14 @@ run_task(TaskConfig(model='Qwen/Qwen2.5-0.5B-Instruct', datasets=['gsm8k'], limi | Handler function | `handle_` prefix | | Benchmark adapter file | `_adapter.py` | -**flake8 ignore list** (`setup.cfg`): `F401, F403, F405, F821, W503, E251, W504, F824, F541, E501, E226, E121-E129, E131, E741`. Do not expand — new ignores must be justified in the PR. +**Ruff ignore list** (`pyproject.toml`): `E501, F401`. Do not expand — new ignores must be justified in the PR. ## Design rules - **Early returns** over nested conditionals. - **Minimal changes**: only touch code related to the current task; no drive-by cleanup. - **Pydantic-first**: cross-module data contracts use Pydantic models. Use `TaskConfig` / `Arguments` for configuration — never raw dicts at module boundaries. +- **Web API responses**: successful JSON responses consumed by the dashboard use models from `evalscope/service/api_models/` and `json_response()`. Regenerate frontend contracts with `cd evalscope/web && npm run contracts:generate`; never hand-edit generated artifacts or add parallel response schemas. - **Reuse existing patterns**: new benchmarks / models / metrics go through existing registries and adapter base classes — no parallel mechanisms. - **DRY** but don't over-abstract just to remove minor duplication. @@ -100,10 +111,11 @@ Don't try to learn the architecture from this file — read these and grep: | Model layer | `evalscope/api/model/model.py`, `evalscope/models/model_apis.py` | | CLI dispatch | `evalscope/cli/` | | Cache schema | `evalscope/api/evaluator/cache.py` | +| Web API response contracts | `evalscope/service/api_models/`, `evalscope/service/responses.py`, `evalscope/web/src/api/generated/` | **Registry decorators**: `@register_benchmark`, `@register_model_api`, `@register_metric`, `@register_aggregation`, `@register_filter`, `@register_evaluator`. -**Adapter base classes** (extend, don't reinvent): `DefaultDataAdapter`, `MultiChoiceAdapter`, `VisionLanguageAdapter`, `Text2ImageAdapter`, `ImageEditAdapter`, `NERAdapter`, `AgentAdapter`. Optional capabilities via mixins: `LLMJudgeMixin`, `SandboxMixin`. +**Adapter base classes** (extend, don't reinvent): `DefaultDataAdapter`, `MultiChoiceAdapter`, `VisionLanguageAdapter`, `Text2ImageAdapter`, `ImageEditAdapter`, `NERAdapter`, `AgentAdapter`. Optional capabilities via mixins: `LLMJudgeMixin`, `CodeExecutionSandboxMixin`. **Non-native backends** live under `evalscope/backend/` (OpenCompass, VLMEvalKit, RAGEval) and are dispatched from `run.py` with their own BackendManager. @@ -111,9 +123,33 @@ Don't try to learn the architecture from this file — read these and grep: 1. Create `evalscope/benchmarks//_adapter.py`. 2. Extend `DefaultDataAdapter`, override `record_to_sample()` (and optionally `sample_to_fewshot()`, `extract_answer()`). -3. Decorate with `@register_benchmark(BenchmarkMeta(name=..., ...))`. -4. Auto-discovered by globbing `evalscope/benchmarks/*/**/*_adapter.py`. -5. Add a smoke test. +3. Reuse the standard dataset flow (`load_subset()` and existing `DataLoader` implementations) for shuffle, limit, repeats, filtering, conversion, and indexing. Override the full `load()` flow only when the standard loaders cannot represent the source format, and keep custom loading limited to benchmark-specific parsing or validation. +4. Use `download_dataset_file()` or `download_dataset_snapshot()` for benchmark media and raw files; do not duplicate hub resolution, cache, path-safety, or download state inside an adapter. +5. Decorate with `@register_benchmark(BenchmarkMeta(name=..., ...))`. +6. Auto-discovered by globbing `evalscope/benchmarks/*/**/*_adapter.py`. +7. Add a smoke test. + +### Evaluation versioning + +`BenchmarkMeta.evaluation_version` is the published version of a benchmark's evaluation semantics. New benchmarks +must declare their initial version explicitly. Raise the minor version when data, sample conversion, default prompt, +choice/target mapping, default scoring/judge, or aggregation semantics change; raise the major version for a rename +or task-definition replacement. Documentation, tests, and pure refactors do not change it. + +## Adding a judge-scored benchmark + +An adapter must **never** call `self.llm_judge.judge()` or parse a judge reply itself — that debt is fenced off by `tests/api/judge/test_gates.py`, which scans every file under `evalscope/benchmarks/` (helpers included, so moving a parser into `utils.py` does not evade it). Score through the JSON output contract in `evalscope/api/judge/` instead: + +1. Pick a `scoring_policy` (`JUDGE_ONLY` or `JUDGE_DEFAULT`). Judge scoring always goes through the contract; there is no opt-in flag and no legacy path. +2. **Single verdict per sample:** implement `judge_definition(context)` and return `JudgeDefinition.labels(...)` for a label mapping or `JudgeDefinition.numeric(...)` for a 0-1 rating. A generic `prompt_template` must state grading criteria only: `OutputContract.instruction()` appends the reply format. An adapter that preserves an official fixed output template may keep that format instruction instead, provided its `OutputContract` schema matches the official template. +3. **Custom shape (multiple cases, ratings, rubrics):** `judge_definition(context)` declares a Pydantic `schema_model`, wraps it in `OutputContract`, and returns `JudgeDefinition.workflow(...)`: + - `cases` contains one `JudgeCase(case_id, output_contract, metadata)` per thing to judge. + - `request(case, placement, completed, context)` renders messages and appends `case.output_contract.instruction()` so the prompt and parser cannot drift. An official fixed output template may be used instead when its required fields and constraints match the case's `OutputContract` schema. + - `reduce(verdicts, context)` folds parsed verdicts into `{metric: value}`. Read a verdict's context from `CaseVerdict.metadata`, never by parsing `case_id`. + - Optional `expand`, `fallback`, and `finalize` callbacks handle staged cases, rule fallbacks, and score finalization. They may be nested functions or private adapter helpers, but are passed only through the returned definition. +4. **Rule short-circuit:** if deterministic scoring settles the sample before judge I/O, return `JudgeDefinition.skip(score, reason='...')`. The non-empty reason is persisted in `Score.metadata` as `judge_skipped=True` and `judge_skip_reason`; the web review panel displays it as rule-based scoring. +5. The executor owns request execution, position swap, repeats, multi-judge aggregation and fail-closed exclusion. Transport retries belong to the model implementation; a reply that fails the contract is not automatically retried and excludes the sample from the metric — never scored 0 or full credit — so a metric's `num` can be below the sample count. +6. Add a scripted-judge test in `tests/api/judge/test_migrated_adapters.py` covering: a valid verdict, a parse failure (prose / malformed), and a transport `[ERROR]` — each must exclude, not silently score. A judge double must carry the surface the definition reads (`score_type`, `score_mapping`, `build_prompt`), and be injected through the `llm_judge` setter rather than a private attribute. ## Conventions & gotchas @@ -122,7 +158,7 @@ Don't try to learn the architecture from this file — read these and grep: - `repeats`: duplicates items for k-metrics. `generation_config.n` is deprecated and mapped. - Use `generation_config` for runtime params. `TaskConfig.timeout` / `stream` are deprecated — forwarded with a warning. - `dataset_args` merges into `BenchmarkMeta._update()` (supports `local_path`, `filters` OrderedDict prepended). -- Models are memoized by `(name, config, base_url, api_key, args)`. +- Models are memoized by `(name, eval_type, config, base_url, api_key, args)`. - Use `@thread_safe` for model creation, `run_in_threads_with_progress` for concurrent eval. - Outputs land in `outputs//{logs,predictions,reviews,reports,configs}/` (see `OutputsStructure`). `use_cache` resumes runs; `rerun_review` recomputes scores only. - `evalscope app` CLI command is **deprecated** (see `evalscope/cli/start_app.py`) — use `evalscope service` for the Web dashboard. @@ -131,6 +167,6 @@ Don't try to learn the architecture from this file — read these and grep: ```bash make dev # once -make lint # before every commit +make lint # apply fixes and run all checks before every commit pytest tests/cli/test_all.py::TestRun::test_ci_lite -v -s -p no:warnings ``` diff --git a/evalscope/CONTRIBUTING.md b/evalscope/CONTRIBUTING.md index df66740..4bdb818 100644 --- a/evalscope/CONTRIBUTING.md +++ b/evalscope/CONTRIBUTING.md @@ -99,6 +99,20 @@ The dev server runs at `http://localhost:5173` and automatically proxies `/api/v **Tech stack:** React 19 · TypeScript · Vite · Tailwind CSS 4 · React Router · Plotly.js +#### Web API response contracts + +Backend Pydantic models in `evalscope/service/api_models/` are the single source of truth for successful JSON responses consumed by the dashboard. Route handlers validate those payloads with `json_response()` before serialization; the frontend uses generated TypeScript types rather than hand-written response schemas. + +After changing a Web API response model, regenerate and commit both generated artifacts: + +```bash +cd evalscope/web +npm run contracts:generate +npm run contracts:check +``` + +Do not edit `src/api/generated/contracts.ts` or `contracts.schema.json` by hand. `make web-contracts-check` performs the same drift check from the repository root and is part of the release build. Error responses and non-JSON responses such as HTML reports and media files remain outside this generated response contract. + ### Full-Stack Development For the best development experience, run both servers simultaneously: @@ -306,18 +320,21 @@ evalscope service This project uses **pre-commit** with the following hooks: -- **flake8** — Python style checker -- **isort** — Import sorting -- **yapf** — Code formatting -- Trailing whitespace, YAML checks, line ending fixes +- **Ruff check** — Python linting (`E`, `F`, and `W`) and import sorting (`I`) +- **Ruff format** — Python code formatting with 120-character lines and single quotes +- Trailing whitespace, YAML checks, and line ending fixes + +Ruff's lint hook runs before its formatter so that any automatic fixes are formatted consistently. Pre-commit is installed by `make dev` with the version pinned in `requirements/dev.txt`. ```bash -# Run all checks +# Apply safe fixes, format maintained Python files, and run all repository checks make lint # or pre-commit run --all-files ``` +If pre-commit modifies files, review and stage those changes, then run `make lint` again. The configured Ruff scope and exclusions are defined in `pyproject.toml`. + ### Testing ```bash @@ -342,9 +359,9 @@ pytest tests/benchmark/test_xxx.py git commit -m "feat: add MyBenchmark adapter" ``` -3. **Run quality checks** before pushing: +3. **Run quality checks before pushing:** ```bash - pre-commit run --all-files + make lint pytest tests/ ``` diff --git a/evalscope/DESIGN.md b/evalscope/DESIGN.md index 23ddd7f..e3e79e2 100644 --- a/evalscope/DESIGN.md +++ b/evalscope/DESIGN.md @@ -4,7 +4,7 @@ colors: # Brand & Accent — IDENTICAL across both themes. Violet is the brand constant. accent: "#816DF8" accent-dark: "#5B3FD6" - accent-dim: "rgba(129,109,248,0.12)" + accent-dim: "rgba(129, 109, 248, 0.12)" purple: "#a78bfa" # Surface ladder (sunken → elevated) — DARK @@ -12,7 +12,7 @@ colors: bg-deep: "#09091a" bg-card: "#12122b" bg-card2: "#16163a" - surface-glass: "rgba(18,18,43,0.7)" + surface-glass: "rgba(18, 18, 43, 0.7)" # Text (3-step ladder) — DARK text: "#e2e8f0" @@ -21,9 +21,9 @@ colors: on-filled: "#ffffff" # Hairline borders — DARK (translucent violet, the near-black bg lets even 10% read) - border: "rgba(129,109,248,0.10)" - border-md: "rgba(129,109,248,0.18)" - border-strong: "rgba(129,109,248,0.28)" + border: "rgba(129, 109, 248, 0.10)" + border-md: "rgba(129, 109, 248, 0.18)" + border-strong: "rgba(129, 109, 248, 0.28)" # Semantic states success: "#10b981" @@ -42,26 +42,26 @@ colors: # ──────────────────────────────────────────────────────────────── # Surface ladder — warm-cream, sunken → elevated - bg-light: "#faf9f5" # warm cream canvas — was cool #f5f6fa - bg-deep-light: "#f0ebe1" # input wells, one step below canvas — was cool #e8eaf2 - bg-card-light: "#ffffff" # pure white — strongest possible contrast against cream canvas - bg-card2-light: "#f5f0e7" # hover / elevated — warm cream-soft, was cool #eef0f7 - surface-glass-light: "rgba(250,249,245,0.80)" # warm-tinted glass — was pure white + bg-light: "#faf9f5" # warm cream canvas — was cool #f5f6fa + bg-deep-light: "#f0ebe1" # input wells, one step below canvas — was cool #e8eaf2 + bg-card-light: "#ffffff" # pure white — strongest possible contrast against cream canvas + bg-card2-light: "#f5f0e7" # hover / elevated — warm cream-soft, was cool #eef0f7 + surface-glass-light: "rgba(250, 249, 245, 0.80)" # warm-tinted glass — was pure white # Accent (unchanged from dark — violet is the brand constant) accent-light: "#6c57e8" - accent-dim-light: "rgba(108,87,232,0.14)" # slightly stronger on white card + accent-dim-light: "rgba(108, 87, 232, 0.14)" # slightly stronger on white card # Text — warm-ink ladder - text-light: "#141413" # warm near-black — was cool #1a1f2e - text-muted-light: "#6c6a64" # warm grey — was cool #5a6378 - text-dim-light: "#8e8b82" # warm grey — was cool #7c8497 + text-light: "#141413" # warm near-black — was cool #1a1f2e + text-muted-light: "#6c6a64" # warm grey — was cool #5a6378 + text-dim-light: "#8e8b82" # warm grey — was cool #7c8497 # Hairlines — SOLID warm hex, not translucent violet. Three concrete tones. # Critical: translucent violet at 0.20 alpha composites to near-invisible # on white cards. Solid warm-grey gives every card a definite boundary. - border-light: "#e6dfd8" # standard hairline — was rgba(violet,0.20) - border-md-light: "#d6cdbe" # emphasized — was rgba(violet,0.30) + border-light: "#e6dfd8" # standard hairline — was rgba(violet,0.20) + border-md-light: "#d6cdbe" # emphasized — was rgba(violet,0.30) border-strong-light: "#c1b6a3" # hover / focus boundary — was rgba(violet,0.40) # Compare slot accents (per-model tagging in compare view) @@ -115,7 +115,7 @@ typography: lineHeight: 1.4 table-xs: fontFamily: System Sans - fontSize: 10px + fontSize: 12px fontWeight: 600 letterSpacing: 0.05em textTransform: uppercase @@ -155,11 +155,11 @@ fontFamily: mono: 'ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, "Liberation Mono", "DejaVu Sans Mono", "Courier New", monospace' rounded: none: 0px - xs: 4px - sm: 8px - md: 12px - lg: 16px - xl: 20px + xs: "4px" + sm: "8px" + md: "12px" + lg: "16px" + xl: "20px" full: 9999px spacing: xs: 4px @@ -173,31 +173,27 @@ spacing: 5xl: 64px shadows: # Dark theme — single deep drop (works on near-black surfaces). - sm: "0 2px 8px rgba(0,0,0,0.4)" - md: "0 4px 20px rgba(0,0,0,0.55)" - lg: "0 8px 40px rgba(0,0,0,0.6)" - glow: "0 0 20px rgba(129,109,248,0.25)" - glow-soft: "0 0 12px rgba(129,109,248,0.2)" + sm: "0 2px 8px rgba(0, 0, 0, 0.4)" + md: "0 4px 20px rgba(0, 0, 0, 0.55)" + lg: "0 8px 40px rgba(0, 0, 0, 0.6)" + glow: "0 0 20px rgba(129, 109, 248, 0.25)" + glow-soft: "0 0 12px rgba(129, 109, 248, 0.20)" # Light theme — two-stop stacks tinted with warm-ink (matches text colour), # not slate. Slate-tinted drops on cream read as a cool-grey smudge and break # the warm canvas. Warm-ink stays consistent with the rest of the palette. - sm-light: "0 1px 2px rgba(20,20,19,0.04), 0 4px 12px rgba(20,20,19,0.06)" - md-light: "0 4px 16px rgba(20,20,19,0.07), 0 12px 32px rgba(20,20,19,0.05)" - lg-light: "0 12px 24px rgba(20,20,19,0.09), 0 24px 48px rgba(20,20,19,0.07)" - glow-light: "0 0 20px rgba(108,87,232,0.22)" - glow-soft-light: "0 0 12px rgba(108,87,232,0.18)" + sm-light: "0 1px 2px rgba(20, 20, 19, 0.04), 0 4px 12px rgba(20, 20, 19, 0.06)" + md-light: "0 4px 16px rgba(20, 20, 19, 0.07), 0 12px 32px rgba(20, 20, 19, 0.05)" + lg-light: "0 12px 24px rgba(20, 20, 19, 0.09), 0 24px 48px rgba(20, 20, 19, 0.07)" + glow-light: "0 0 20px rgba(108, 87, 232, 0.22)" + glow-soft-light: "0 0 12px rgba(108, 87, 232, 0.18)" gradients: brand: "linear-gradient(135deg, #816DF8 0%, #a78bfa 100%)" accent: "linear-gradient(135deg, #0F9C7E 0%, #06b6d4 100%)" surface: "linear-gradient(135deg, rgba(129,109,248,0.08) 0%, rgba(167,139,250,0.05) 100%)" - kpi-0: "linear-gradient(135deg, #6366f1, #8b5cf6)" - kpi-1: "linear-gradient(135deg, #10b981, #06b6d4)" - kpi-2: "linear-gradient(135deg, #f59e0b, #f97316)" - kpi-3: "linear-gradient(135deg, #ec4899, #8b5cf6)" nav-hairline: "linear-gradient(90deg, transparent 0%, #816DF8 50%, transparent 100%)" transition: fast: "150ms cubic-bezier(0.4, 0, 0.2, 1)" - base: "180ms ease" + base: "250ms cubic-bezier(0.4, 0, 0.2, 1)" slow: "400ms cubic-bezier(0.4, 0, 0.2, 1)" breakpoints: sm: 640px @@ -216,7 +212,9 @@ score-formula: # Design System: EvalScope Console -## Overview +## Principles {#principles} + +> **Addressable section — `principles`.** The design philosophy, brand posture, and dual-theme parity rules that govern every downstream decision. See also the normative *Do's and Don'ts* under [Decision Records](#decisions). EvalScope's web dashboard is a developer-platform brand for **LLM evaluation and benchmarking** — the page is an instrument panel for engineers running evals, written for people who already know the syntax. It earns that posture through **two equally weighted themes** rather than one canonical mode with a translated companion. Both themes share the same vocabulary — same type, same spacing, same radii, same components — but each carries its own surface philosophy. They are two voices of one brand, not one design re-tinted. @@ -226,26 +224,30 @@ EvalScope's web dashboard is a developer-platform brand for **LLM evaluation and The brand constant across both themes is the single violet `{colors.accent}` (`#816DF8` dark / `#6c57e8` light) used for primary CTAs, active nav states, focus rings, and the wordmark accent — plus the dynamic HSL score gradient (`hsl(score × 120, 70%, 45%)`) that maps a 0-1 metric to red → yellow → green. Both signals work over either canvas. Everything else — surface ladder, hairline material, shadow tint, on-canvas text colour — is theme-specific by design, because dark and light surfaces need *different* materials to produce the same hierarchy. -Type is the second decisive voice and is **theme-agnostic**. The brand uses cross-platform system font stacks (no web font is loaded) — `system-ui, -apple-system, "Segoe UI", Roboto, ...` for narrative and `ui-monospace, SFMono-Regular, Menlo, Consolas, "Liberation Mono", ...` for technical labels. Each OS resolves to its own native UI face. Headlines are sentence-case with `tracking-tight` on display numbers; **all-caps + `tracking-wider`** is reserved for tiny section eyebrows (12 px / 10 px), never headlines. Weight ceiling is **700**; the working set is 400 / 500 / 600 / 700. +Type is the second decisive voice and is **theme-agnostic**. The brand uses cross-platform system font stacks (no web font is loaded) — `system-ui, -apple-system, "Segoe UI", Roboto, ...` for narrative and `ui-monospace, SFMono-Regular, Menlo, Consolas, "Liberation Mono", ...` for technical labels. Each OS resolves to its own native UI face. Headlines are sentence-case with `tracking-tight` on display numbers; **all-caps + `tracking-wider`** is reserved for 12 px section eyebrows and table labels, never headlines. Weight ceiling is **700**; the working set is 400 / 500 / 600 / 700. **Key Characteristics:** - **Dual-theme parity, not dual-theme translation.** Dark uses translucent violet hairlines on near-black; light uses solid warm-grey hairlines on cream. They produce the same hierarchy through opposite material choices. Theme is persisted to `localStorage` and applied via `data-theme` on `` before first paint to avoid a flash. - A single violet primary CTA `{colors.accent}` carries every conversion target on both themes, paired with a transparent **ghost** secondary. The brand uses a `{rounded.sm}` 8-px button shape for primary/secondary in the *console* (no marketing pills — this is an in-product surface). - The primary CTA **glows on hover** with violet at 20-25 % alpha on both themes. That violet glow is the brand's signature interaction — identical animation, identical colour, identical timing across themes. -- Every card section title, form label, and table header sets in `{typography.label-xs}` — 12 px (10 px for tables), `font-semibold`, **UPPERCASE**, `tracking-wider`, muted color. Body and titles stay sentence-case. The contrast between these two voices does most of the hierarchy work. +- Every card section title, form label, and table header is at least 12 px, `font-semibold`, **UPPERCASE**, `tracking-wider`, and uses the AA-safe muted color. Body and titles stay sentence-case. The contrast between these two voices does most of the hierarchy work. - A dynamic HSL **score chip** (`hsl(score × 120, 70%, 45%)`) is the second-most-recognizable component after the brand violet — it is how the product communicates pass/fail. Identical formula on both themes; the chip's saturation works over cream and over near-black. - **Light theme uses solid hex hairlines, not translucent violet.** This is the most important light-theme rule, and the one most often broken on first attempt: a violet alpha overlay disappears into a near-white page, so light surfaces require concrete warm-grey edges (`#e6dfd8` / `#d6cdbe` / `#c1b6a3`) to keep their boundaries. -- A complete domain token set exists for **chat bubbles** (5 semantic roles: user / bot / tool / reasoning / system), **compare slots** (3 per-model accent colors), and **KPI gradients** (4 named gradient pairs) — these are first-class brand tokens, not ad-hoc colors. +- A complete domain token set exists for **chat bubbles** (5 semantic roles: user / bot / tool / reasoning / system) and **compare slots** (3 per-model accent colors) — these are first-class brand tokens, not ad-hoc colors. - A glassmorphic sticky top-nav (52 px, 12 px backdrop-blur, 1-px violet-to-transparent gradient hairline along the top edge) is the only "marketing-y" flourish the product allows itself. Dark uses translucent indigo glass; light uses translucent cream glass. -## Colors +## Design Tokens {#tokens} + +> **Addressable section — `tokens`.** The executable token reference: color, typography, layout, elevation, and shape scales. Runtime CSS custom properties in `evalscope/web/src/index.css` are the source of truth. `npm run design:tokens` synchronizes the matching frontmatter values, while the drift check fails CI when the generated documentation is stale. + +### Colors > **Note on dual theme.** Every color token below has a *dark* (default) and *light* value. Hex pairs are listed as `dark / light`. Components reference tokens by name — never by raw hex — so theme switching is free. > > **Light values use SOLID hex for hairlines, not translucent violet.** This is the structural difference from earlier light-theme generations and the single most-broken light-theme rule. See `{colors.border-light}` below and *Elevation & Depth* for the reason. -### Brand & Accent +#### Brand & Accent The accent family is **identical in spirit on both themes** — a single violet handles every conversion target. Slightly different hex values per theme (the dark violet is brighter to read on near-black; the light violet is a half-step deeper to hold weight against white cards) but the same brand voltage. @@ -255,7 +257,7 @@ The accent family is **identical in spirit on both themes** — a single violet - **Violet Mist** (`{colors.accent-dim}` — `rgba(129,109,248,0.12)` / `rgba(108,87,232,0.14)`): The low-alpha violet used as pill background and focus-ring fill. Light theme runs slightly stronger (0.14 vs 0.12) because white cards need a touch more saturation to read the mist. - **Violet Glow** (`{shadows.glow}` — `0 0 20px rgba(129,109,248,0.25)` / `0 0 20px rgba(108,87,232,0.22)`): The signature hover halo on primary buttons and active nav. Same effect, same magnitude, on both themes. -### Surface +#### Surface Each theme operates with a 4-step surface ladder, sunken-to-elevated. The **ladder structure is shared**; the **material is different** — dark walks an indigo ladder, light walks a warm-cream ladder. The semantic of each step is the same: `bg-deep` is *below* the page, `bg-card` is the working surface, `bg-card2` is the elevated state. @@ -265,20 +267,20 @@ Each theme operates with a 4-step surface ladder, sunken-to-elevated. The **ladd - **Card Elevated** (`{colors.bg-card2}` — `#16163a` / `#f5f0e7`): Hover state for clickable cards and rows; also the inactive-tab fill in pill-tab containers. Light theme's elevated state is warm-cream-soft — the elevated state is darker on dark theme but lighter-than-card-but-warmer on light theme (the white card with a soft-cream hover reads as "depressed into the cream canvas"). - **Glass** (`{colors.surface-glass}` — `rgba(18,18,43,0.7)` / `rgba(250,249,245,0.80)`): Translucent surface for the sticky top-nav, used with a 12-px backdrop-blur. Light theme uses tinted cream glass (matches the canvas), NOT pure white — white glass on cream reads as a foreign sheet floating in space. -### Text +#### Text - **Ink** (`{colors.text}` — `#e2e8f0` / `#141413`): All headings, body, table cell values, button labels on non-filled surfaces. Light theme uses warm-near-black (`#141413`, ≈ the same value Claude.com uses) rather than a cool slate (`#1a1f2e`), so the text temperature matches the canvas temperature. - **Muted** (`{colors.text-muted}` — `#8896aa` / `#6c6a64`): Secondary labels, nav-link inactive text, card-header micro-labels, button "ghost" idle text. *This is also the color section-eyebrow uppercase labels are set in.* Light theme uses warm-grey (`#6c6a64`) rather than cool-slate (`#5a6378`) to stay coherent with the warm canvas. - **Dim** (`{colors.text-dim}` — `#7a8195` / `#8e8b82`): Lowest-priority text — placeholder text, timestamps in compact rows, table empty-state. **Contrast tuned to ~3.6 : 1** against `{colors.bg-card}` on both themes — sits just above the WCAG AA Large floor (3 : 1), still **below AA Normal (4.5 : 1)**. ⚠️ Reserve for ≥ 14 px non-essential metadata. Light theme uses a warm-grey at the same luminance step as the dark theme's cool-grey — the perceived hierarchy stays identical. - **On Filled** (`{colors.on-filled}` — `#ffffff` / `#ffffff`): Text on `{colors.accent}` and other saturated fills. Identical on both themes — the violet CTA is dark enough on both that white text holds. -### Hairlines (the structural difference between themes) +#### Hairlines (the structural difference between themes) - **Border** (`{colors.border}` — `rgba(129,109,248,0.10)` / `#e6dfd8`): The default 1-px card / input / divider boundary. **Dark uses translucent violet at 10 % alpha** because the near-black bg-to-card luminance step already does most of the boundary work — the violet hairline just tints it. **Light uses a SOLID warm-grey hex** (`#e6dfd8`, Claude-style cream-hairline) because the white-card-on-cream luminance step is gentle enough that a translucent violet overlay disappears into the page. Borders on light theme are concrete materials, not tints. - **Border Emphasized** (`{colors.border-md}` — `rgba(129,109,248,0.18)` / `#d6cdbe`): One step stronger — used on form inputs after focus, on the active-state of hover cards, on the boundary between a card and a nested section. - **Border Strong** (`{colors.border-strong}` — `rgba(129,109,248,0.28)` / `#c1b6a3`): The strongest boundary — used by `{components.card-hover}` on hover lift, and by elevated cards in modal contexts. On both themes this is the "this thing is grabbing attention" hairline. -### Semantic +#### Semantic - **Success** (`{colors.success}` — `#10b981` / `#059669`) / **Success Bg** (`rgba(16,185,129,0.08)`) / **Success Border** (`rgba(16,185,129,0.20)`): Confirmed / passed states; success toasts; chat-bot bubble border. - **Warning** (`{colors.warning}` — `#f59e0b` / `#d97706`) / **Warning Bg** (`rgba(245,158,11,0.08)`) / **Warning Border** (`rgba(245,158,11,0.20)`): Pending / caution; tool-call chat bubbles. @@ -286,7 +288,7 @@ Each theme operates with a 4-step surface ladder, sunken-to-elevated. The **ladd - **Info** (`{colors.info}` — `#60a5fa` / `#3b82f6`): Latency chart series, informational toasts. - **Pass** (`{colors.pass}` — `rgb(45,104,62)` / `rgb(16,108,55)`) / **Fail** (`{colors.fail}` — `rgb(151,31,44)` / `rgb(180,30,42)`): Deep saturated greens / crimsons for boolean pass/fail badges where the tone needs more weight than the soft semantic family. -### Score Gradient (Signature) +#### Score Gradient (Signature) The product's emotional core. A 0-1 score maps to **`hsl(score × 120, 70%, 45%)`**: 0 → red, 0.5 → yellow, 1 → green. Used as both foreground and translucent background on score chips, dataset chips, and group-header best-score callouts. This is computed inline (`scoreColor` / `scoreBg` helpers), never stored as a static palette. **Treat the formula as a brand asset** — do not reskin to a 5-step bucket, do not introduce a 4th hue. @@ -302,7 +304,7 @@ Foreground uses HSL for predictable hue progression; background uses an RGB-inte **Score Ring** (`{components.score-ring}` — SVG circular progress used in `` and the "Overall Score" callout in ``): the active arc is `stroke={scoreColor(score)}`. Stroke width must be **≥6 px** for the 48 × 48 mini ring and **8 px** for the 72 × 80 summary ring — anything thinner reduces the colored area to the point where the mid-hue olive stops carrying. Background arc uses `var(--border)` for a neutral track. -### Compare Slots +#### Compare Slots Three per-model accent colors used to tag side-by-side model comparisons. Each slot has a `dot`, `border`, `bg`, and `bg-header` tint at ~10-30 % alpha: @@ -312,7 +314,7 @@ Three per-model accent colors used to tag side-by-side model comparisons. Each s If a comparison view exceeds 3 models, *do not invent a 4th brand color* — collapse into a numbered legend instead. -### Chat Bubble Roles +#### Chat Bubble Roles Five semantic roles, each with a complete 7-token set (`bg`, `bg-hl`, `border`, `border-hl`, `icon-bg`, `icon-border`, `color`): @@ -324,20 +326,15 @@ Five semantic roles, each with a complete 7-token set (`bg`, `bg-hl`, `border`, Bubble containers are `{rounded.md}` with the role's tint background and border; hover/highlight states use the `*-hl` variants. -### KPI Gradients +#### KPI Icon Tile -Four named linear gradients for the four hero KPI tiles on the dashboard: +The four hero KPI tiles on the dashboard share one hue: the 40 × 40 `{rounded.md}` icon tile is filled with `{colors.accent-dim}` and its glyph inked in `{colors.accent}`, on both themes. -- **Indigo→Violet** (`{gradients.kpi-0}` — `linear-gradient(135deg, #6366f1, #8b5cf6)`) -- **Emerald→Teal** (`{gradients.kpi-1}` — `linear-gradient(135deg, #10b981, #06b6d4)`) -- **Amber→Orange** (`{gradients.kpi-2}` — `linear-gradient(135deg, #f59e0b, #f97316)`) -- **Pink→Violet** (`{gradients.kpi-3}` — `linear-gradient(135deg, #ec4899, #8b5cf6)`) +**Do not introduce a per-KPI hue.** The four counters are the same kind of quantity, so distinct hues would assert a distinction that does not exist, and would put the loudest colour in the app on its least specific numbers. A hue is reserved for values that carry a meaning, such as `{chart.*}` for perf-metric series or the score scale. -Always applied to the 40 × 40 `{rounded.md}` icon tile inside a `{components.kpi-card}`. **Same gradient values on both themes** — they're saturated enough to work over either canvas. +#### Chart Palette (Perf Metrics) -### Chart Palette (Perf Metrics) - -Four hue tokens used to mark perf-metric series (latency / TTFT / TPOT / token-usage) across the KPI strip, the chart series legends, and the percentile-table accent headers in ``. **The two themes use different RGB values for the same hue** — unlike the KPI gradients, these aren't shared across themes: +Four hue tokens used to mark perf-metric series (latency / TTFT / TPOT / token-usage) across the KPI strip, the chart series legends, and the percentile-table accent headers in ``. **The two themes use different RGB values for the same hue:** - **Latency** (`{chart.latency}` — `#60a5fa` dark / `#2563eb` light) - **TTFT** (`{chart.ttft}` — `#34d399` dark / `#047857` light) @@ -348,15 +345,15 @@ Four hue tokens used to mark perf-metric series (latency / TTFT / TPOT / token-u **KPI strip surface**: the strip in `` uses `{colors.bg-card}` (matching the outer card), not `{colors.bg-deep}`. On light theme, `{colors.bg-deep}` is even warmer than the cards and pushes the chart hues into the warm-on-warm range — losing the contrast that makes the colored numbers carry. Visual separation comes from the border + dividers, not bg differentiation. -### Brand Gradients (Decorative) +#### Brand Gradients (Decorative) - **Brand** (`{gradients.brand}` — `linear-gradient(135deg, #816DF8 → #a78bfa)`): For `gradient-text` and large brand moments. - **Accent** (`{gradients.accent}` — `linear-gradient(135deg, #0F9C7E → #06b6d4)`): For the optional emerald-to-cyan accent text. - **Surface** (`{gradients.surface}` — `linear-gradient(135deg, rgba(129,109,248,0.08) → rgba(167,139,250,0.05))`): The subtle violet wash layered behind KPI cards via `::before`. -## Typography +### Typography -### Font Family +#### Font Family Two cross-platform **system font stacks** carry the entire system — each OS resolves to its own native UI face. No `@font-face` is loaded; this is deliberate. Each stack starts with the modern CSS `system-ui` / `ui-monospace` generic family and falls back to named faces for older browsers and per-OS targets: @@ -365,23 +362,23 @@ Two cross-platform **system font stacks** carry the entire system — each OS re Antialiasing is forced (`-webkit-font-smoothing: antialiased`). No web font is loaded — the brand reads as "native developer tool, not marketing site" precisely because of this. The trade-off is per-OS rendering variance; if pixel-identical screenshots across platforms are required, see *Note on Font Substitutes* below. -### Hierarchy +#### Hierarchy | Token | Size | Weight | Tracking | Use | |---|---|---|---|---| -| `{typography.display-xl}` | 24px | 700 | tight | KPI value, hero number (dashboard `{components.kpi-card}`). | +| `{typography.display-xl}` | 24px | 700 | tight | Hero number where a single figure is the whole message. | | `{typography.title-md}` | 16px | 700 | normal | Card-title model name, group-header titles, brand wordmark. | | `{typography.body-sm}` | 14px | 400 / 500 | normal | Default body text, button-md label, table-cell text, paragraph copy. | | `{typography.label-xs}` | 12px | 600 | wider | **UPPERCASE** card-section header, form label, badge text — the brand's signature eyebrow. | | `{typography.body-xs}` | 12px | 400 / 500 | normal | Empty-state hint, pill / badge body, mobile-nav link. | -| `{typography.table-xs}` | 10px | 600 | wider | **UPPERCASE** table-header micro-text — whispers, doesn't shout. | +| `{typography.table-xs}` | 12px | 600 | wider | **UPPERCASE** table-header text; never smaller because headers are essential. | | `{typography.caption-mono}` | 12px | 400 (mono) | normal | Timestamps, score values, dataset names in chips. | | `{typography.code}` | 13–14px | 400 (mono) | normal | Log viewer, JSON viewer, terminal-style output. | | `{typography.button-sm}` | 12px | 500 | normal | `{components.button}` `size="sm"`. | | `{typography.button-md}` | 14px | 500 | normal | `{components.button}` `size="md"` (default). | | `{typography.button-lg}` | 16px | 500 | normal | `{components.button}` `size="lg"` — for hero callouts only. | -### Principles +#### Principles - **UPPERCASE + `tracking-wider` is the eyebrow voice — never the headline voice.** It marks "this introduces a region." Card titles, model names, and KPI labels stay sentence-case (or lower-case for the brand wordmark's lowercase "v"). - **`tracking-tight` is reserved for the display tier** — KPI numbers and the brand wordmark only. It tells the reader "this is set big on purpose." @@ -390,16 +387,16 @@ Antialiasing is forced (`-webkit-font-smoothing: antialiased`). No web font is l - **Line-heights inherit Tailwind defaults** (1.5 for body, 1.25 for headings). Don't override globally; rely on padding for vertical rhythm in tight stacks (cards, chips). - **No web font.** The system stack is the system. Loading Inter or Geist on top would break the "native console" feel. -### Note on Font Substitutes +#### Note on Font Substitutes EvalScope uses the OS-native stack, so there are no proprietary faces to substitute. If a future skin needs to enforce a *single* face across all OSes for screenshot consistency: - **Sans substitute** — *Inter* (400 / 500 / 600 / 700) is the closest stylistic match to the SF-on-macOS rendering; preserves the geometric / neutral character. - **Mono substitute** — *JetBrains Mono* (400) at 12–14 px matches the technical voice well; *IBM Plex Mono* is a close second. -## Layout +### Layout -### Spacing System +#### Spacing System - **Base unit**: 4 px (Tailwind's default scale). - **Tokens** (Tailwind-aligned): @@ -409,7 +406,7 @@ EvalScope uses the OS-native stack, so there are no proprietary faces to substit - **Inline gap**: `{spacing.md}` (12 px) for component rows inside a card, `{spacing.xl}` (20 px) for inter-section gaps between major page blocks. - **Pill / chip gap**: `{spacing.sm}` (6-8 px) — tight, scan-friendly. Pills are meant to wrap. -### Grid & Container +#### Grid & Container - **Max width**: `1600px` (`max-w-[1600px]`). Centered (`mx-auto`). Wide enough for 4-up KPI strip + side-by-side comparison; narrow enough that line-length never sprawls. - **Column patterns**: @@ -419,7 +416,7 @@ EvalScope uses the OS-native stack, so there are no proprietary faces to substit - **Form pairs**: `grid-cols-1 md:grid-cols-2` for label-value pairs. - **Gutters**: 16 px horizontal at all sizes. -### Whitespace Philosophy +#### Whitespace Philosophy Whitespace separates the *bands* — not the components inside a band. Section spacing is generous (`flex flex-col gap-5` → 20 px between major blocks); card interiors are tight (`gap-2` / `gap-3` between rows inside a card). The page reads as engineered — *large gaps + tight interior, never the other way around*. The dark page lets cards float visually without needing margin to assert themselves; the hairline border does the bordering work. @@ -428,9 +425,9 @@ The brand's voice is **information-rich but uncluttered** — a typical dashboar 2. UPPERCASE eyebrows visually section the layout without horizontal rules. 3. Score chips compress a percentage + benchmark name + color signal into 6 characters of mono. -### Responsive Strategy +#### Responsive Strategy -#### Breakpoints (Tailwind defaults) +##### Breakpoints (Tailwind defaults) | Name | Width | Key Changes | |---|---|---| @@ -440,11 +437,13 @@ The brand's voice is **information-rich but uncluttered** — a typical dashboar | Desktop | 1024–1279px | Full pill-style nav with icon + label; KPI strip goes 4-up. | | Wide | ≥ 1280px | Container holds at `max-w-[1600px]`; bands stretch but content centers. | -#### Touch Targets +##### Touch Targets -The top-nav icon-only buttons (tablet) are 32 × 32 — *under the 44 × 44 WCAG floor*. This is a known compromise for the developer-tool density; on actual touch devices, hit areas are extended via padding. Primary buttons reach ~36 px tall in `md` size and ~44 px in `lg` — meet the floor at `lg`. +**Normative rule (executable, enforced — see [Component Contracts](#component-contracts)).** Every primary `Pointer_Target` — navigation, mobile menu, compare-selection, and disclosure controls — MUST expose a hit area of **≥ 44 × 44 CSS px on coarse pointers**. This is a hard floor asserted by the E2E/axe suite at the 390 px viewport, **not** a documented known-failure or accepted compromise. -#### Collapsing Strategy +The *visual* chrome may be smaller than the *hit area*, and only the hit area is governed by the rule. The top-nav icon-only buttons (tablet) render a 32 × 32 visual box, but on coarse pointers their tappable region is expanded to ≥ 44 × 44 via symmetric padding / an `::before` hit-area overlay — the icon stays 32 × 32, the target does not shrink below the floor. Primary buttons reach ~36 px tall in `md` and ~44 px in `lg`; where an `md` button is a primary `Pointer_Target` on touch, it is promoted to the 44 px hit area by the same rule. + +##### Collapsing Strategy - **Nav**: Desktop = pill-row with icon + label; tablet = icon-only pills; mobile = logo + hamburger toggling a stacked drop-down (`max-height` animated, 300 ms). - **KPI strip**: 4-up → 2-up below `lg`. Each tile keeps its `{rounded.md}` 12-px shape and 20-px padding. @@ -452,13 +451,13 @@ The top-nav icon-only buttons (tablet) are 32 × 32 — *under the 44 × 44 WCAG - **Forms**: Two-column label/value at `md+`, single column below. - **Table**: Horizontal scroll wrapper preserves all columns rather than dropping them. The card-shell border keeps the scroll area framed. -#### Image / Icon Behavior +##### Image / Icon Behavior - **Iconography**: `lucide-react`, almost always 14–18 px, color inherits `currentColor`. The only non-Lucide mark is the brand SVG in the top-nav (a hand-drawn triangle with an amber check, rendered with `currentColor` so it follows theme). - **No marketing imagery**: This is a console — no hero photo, no customer-logo strip, no illustrated empty-states. Empty states use a single Lucide icon in a 64 × 64 `{rounded.lg}` deep-well tile. - **Charts**: Plotly-based, theme-aware — chart series colors are `{colors.chart-*}` tokens (latency / TTFT / TPOT / token). -## Elevation & Depth +### Elevation & Depth | Level | Treatment | Use | |---|---|---| @@ -473,16 +472,16 @@ The top-nav icon-only buttons (tablet) are 32 × 32 — *under the 44 × 44 WCAG **Brand rule (hairlines)**: light theme borders are **SOLID HEX**, not translucent violet at any alpha. The earlier light-theme generation used `rgba(108,87,232,0.20-0.40)` and the violet alpha composited to within a few luminance steps of the white card — borders effectively dissolved, especially on outline buttons (`Go to Index`, `Find msg id`) and on input rings (`Score Threshold` field). The current system uses three concrete warm-grey hex values (`#e6dfd8` / `#d6cdbe` / `#c1b6a3`) — each step is a definite material, not a tint. The dark theme keeps translucent violet at 10-28 % because the indigo-bg-to-card luminance delta is already doing most of the boundary work; light theme has no such luminance assist and must rely on the hairline alone. -### Decorative Depth +#### Decorative Depth - **Backdrop-blur**: 12 px on `{colors.surface-glass}` for the sticky top-nav. This is the only blur effect in the system. - **Hairline gradient line**: The top of the nav draws a 1-px `transparent → {colors.accent} → transparent` line at 40 % opacity. The closest the design comes to "decoration." -- **Card lift**: Hover lifts L1/L2 cards `-2px` (`{components.card-hover}`) or `-3px` (`{components.kpi-card}`), simultaneously upgrading their shadow ladder one step. +- **Card lift**: Hover lifts L1/L2 cards `-2px` (`{components.card-hover}`), simultaneously upgrading their shadow ladder one step. - **Gradient text**: `.gradient-text` and `.gradient-text-accent` utilities apply `{gradients.brand}` / `{gradients.accent}` to text via `background-clip: text`. Use sparingly — reserved for hero brand moments, never for body or table cells. -## Shapes +### Shapes -### Border Radius Scale +#### Border Radius Scale | Token | Value | Use | |---|---|---| @@ -496,14 +495,16 @@ The top-nav icon-only buttons (tablet) are 32 × 32 — *under the 44 × 44 WCAG **No pill (100-px) shape.** Unlike Vercel's marketing pill, EvalScope is an in-product surface; all CTAs use `{rounded.sm}` 8 px. Pills (`{rounded.full}`) are exclusively for *data* — badges, chips, score indicators. -### "Photography" — Iconography Geometry +#### "Photography" — Iconography Geometry - **Brand mark**: Hand-drawn SVG triangle with an amber check; rendered inline at 28 × 25 px with `currentColor` so it follows theme. - **Lucide icons**: 14 px in dense lists, 16 px in nav, 18 px in form controls, 28 px in empty-state hero tiles. Stroke width 1.5–2. -- **KPI icon tile**: 40 × 40, `{rounded.md}`, filled with one of `{gradients.kpi-0..3}`, white icon centered. +- **KPI icon tile**: 40 × 40, `{rounded.md}`, filled with `{colors.accent-dim}`, `{colors.accent}` icon centered. - **Chart**: Plotly canvas, no rounded corners; sits inside a `{rounded.md}` card frame. -## Components +## Components {#components} + +> **Addressable section — `components`.** The component contracts built from the tokens above: buttons, cards, inputs, tabs, navigation, tables, badges, and the signature composite surfaces. Each entry references tokens by name so a re-skin propagates automatically. ### Buttons @@ -533,8 +534,9 @@ Disabled = `opacity: 0.5` + `cursor: not-allowed`. Transitions use `{tokens.tran **`{components.card-hover}`** — utility applied to clickable cards. - Adds `-2 px translateY` on hover, upgrades shadow to `{shadows.lg}` (L4), strengthens border to `{colors.border-strong}`. All transitions in `{tokens.transition}`. -**`{components.kpi-card}`** — the dashboard hero metric tile. -- Same chrome as `{components.card}` but layered with `{gradients.surface}` (subtle violet wash, 5-8 % alpha) via `::before`. Hosts a 40 × 40 `{rounded.md}` icon tile filled with `{gradients.kpi-0..3}`. Hover lifts `-3 px` and ramps to L4 shadow. Stagger-animated on first paint (60 ms steps). +**`{components.kpi-strip}`** — the counter strip that opens a page. +- One joined surface, not a row of free-floating tiles: the counters are the same kind of quantity, so separating them into individually lifting cards would assert a distinction that does not exist. Chrome is `{components.card}` (`{colors.bg-card}` fill, 1-px `{colors.border}`, `{rounded.md}`, `{shadows.sm}`) wrapping hairline-divided cells; the strip clips its corners so the dividers never break the outline. A cell is 20-px padded (`{spacing.xl}`) in the `hero` density and holds an optional 32 × 32 `{rounded.sm}` icon tile filled `{colors.accent-dim}` / inked `{colors.accent}`, a `{typography.title-md}` value and a `{typography.body-xs}` label. Interactive cells are buttons and tint to `{colors.bg-card2}` on hover — the strip itself never lifts or transforms. +- Two denser variants share the same contract: `dense` (in-panel metric overview — 2/3/4-up grid of gap-px cells, 18-px value, 10-px label, optional per-cell accent colour) and `inline` (identity/config row — flexible cells with a 140-px floor that wrap, `{typography.body-sm-strong}` value over a `{typography.table-xs}` uppercase label, no icons). **`{components.card-glass}`** — the glassmorphic surface used by the top-nav. - `{colors.surface-glass}` background + 12-px backdrop-blur. Always combined with a 1-px hairline border. Reserve for sticky-positioned surfaces — diffuse blur is performance-sensitive. @@ -576,7 +578,7 @@ Disabled = `opacity: 0.5` + `cursor: not-allowed`. Transitions use `{tokens.tran **`{components.table}`** — sortable data table. - Wrapped in `{components.card}` chrome (`{rounded.md}` border + bg) with `overflow-x-auto`. -- **Header cells**: `{typography.table-xs}` — 10 px, semibold, UPPERCASE, `tracking-wider`, `{colors.text-dim}`. They *whisper*. Sortable headers show a triple-state chevron (`ChevronsUpDown` idle, `ChevronUp/Down` active) and lift to `{colors.text}` on hover; active sort column turns `{colors.accent}`. +- **Header cells**: `{typography.table-xs}` — 12 px, semibold, UPPERCASE, `tracking-wider`, `{colors.text-muted}`. Headers are essential interpretation text and therefore meet the 12 px/AA floor. Sortable headers show a triple-state chevron (`ChevronsUpDown` idle, `ChevronUp/Down` active) and lift to `{colors.text}` on hover; active sort column turns `{colors.accent}`. - **Row dividers**: 1-px `{colors.border}`. Clickable rows hover-fill `{colors.bg-card2}`. - **Empty state**: Centered, dimmed "No data" cell — no illustration. @@ -594,7 +596,7 @@ Disabled = `opacity: 0.5` + `cursor: not-allowed`. Transitions use `{tokens.tran ### Signature Components -**`{components.kpi-card}`** — see *Cards & Containers* above. +**`{components.kpi-strip}`** — see *Cards & Containers* above. **`{components.eval-run-card}`** — a full-width borderless button-styled card row. - Two rows of content: bold model name + colored score pill on the right; second row has a mono timestamp + a wrap of `{components.score-chip}` entries (one per benchmark). Hover only changes border tint (no lift). @@ -615,12 +617,161 @@ Disabled = `opacity: 0.5` + `cursor: not-allowed`. Transitions use `{tokens.tran **`{components.score-badge}`** — the bold percentage pill at the top of an eval row. - `{rounded.full}`, 10/2 padding, `{typography.body-sm}` bold + `tabular-nums`, HSL-computed fg/bg. Distinct from `{components.score-chip}` by size and weight. +### Component Contracts (Responsive & State) {#component-contracts} + +> **Addressable sub-section — `component-contracts`.** Every responsive component below carries two contracts, and both are **normative and executable** — not descriptive prose: +> +> - a **Responsive contract** — the layout, wrapping, and visibility behavior at each of the five breakpoints (**Mobile < 640 px · Small 640–767 px · Tablet 768–1023 px · Desktop 1024–1279 px · Wide ≥ 1280 px**, per [Responsive Strategy](#tokens)'s breakpoint table), and +> - a **State contract** — the presentation requirement for every visible state (**default · hover · focus · active · disabled · error**). A state a component cannot enter is listed as *n/a* with the reason, so the enumeration stays complete. + +#### Normative contract rules (executable, not aspirational) + +These thresholds are **enforceable rules, not logged known-failures**. The E2E/axe suite (task 19.2) asserts them at the 390 px viewport and in both themes; a violation fails CI. Each component contract below inherits them. + +- **R-TOUCH — Touch-target floor 44 × 44 CSS px.** On coarse pointers, every primary `Pointer_Target` in navigation, the mobile menu, compare-selection, and disclosure controls MUST expose a hit area **≥ 44 × 44 CSS px**, independent of the visual icon/box size (padding or an `::before` overlay carries the extra area). +- **R-CONTRAST — AA contrast floor.** Essential text MUST meet **AA_Contrast in both themes**: **≥ 4.5 : 1** for normal text and **≥ 3 : 1** for large text (≥ 24 px, or ≥ 18.66 px bold). `dim`/`muted` tokens applied to essential content are promoted to clear this floor; non-text focus indicators MUST reach **≥ 3 : 1** against the adjacent background. +- **R-WRAP — No lossy truncation of essential metadata.** Responsive text containers use `break-words` + `min-w-0` and MUST NOT apply `truncate`/ellipsis/overflow-hidden to essential metadata; at 390 px every field label and value stays visible and wraps at word (then character) boundaries with no page-level horizontal scroll. +- **R-FOCUS — Visible focus in both themes.** Every interactive component's **focus** state renders a visible focus indicator (`{colors.accent}` ring + `{colors.accent-dim}` fill) meeting R-CONTRAST's ≥ 3 : 1 non-text floor on both themes. Individual state contracts reference this rule rather than restating it. + +#### `{components.top-nav}` / `{components.nav-link}` / `{components.icon-button}` — navigation + +- **Responsive contract:** + - **Mobile < 640:** logo + hamburger only; nav items collapse into a stacked drop-down (animated `max-height`, 300 ms); GitHub link + locale toggle live inside the drawer. Drawer toggle and every drawer item satisfy R-TOUCH. + - **Small 640–767:** hamburger drawer retained; GitHub icon and locale toggle surface directly in the bar. + - **Tablet 768–1023:** icon-only pill buttons (32 × 32 visual) with `title` tooltips; text labels hidden; hit area ≥ 44 × 44 per R-TOUCH. + - **Desktop 1024–1279:** full pill links with icon + label; active = solid `{colors.accent}` + glow. + - **Wide ≥ 1280:** identical to Desktop; container caps at `max-w-[1600px]`, bar content centers. +- **State contract:** + - **default:** `{colors.text-muted}` label on transparent fill. + - **hover:** label lifts to `{colors.text}`, fill → `{colors.bg-card2}`. + - **focus:** per R-FOCUS. + - **active (current route):** `{colors.accent}` fill, `{colors.on-filled}` label, `{shadows.glow}` halo. + - **disabled:** *n/a* — nav destinations are always actionable. + - **error:** *n/a* — navigation carries no validation state. + +#### `{components.button}` — buttons (primary / ghost / outline) + +- **Responsive contract:** intrinsic width, wraps within its container per R-WRAP; size is chosen by usage, not breakpoint (`sm` ~28 px, `md` ~36 px, `lg` ~44 px). A primary `Pointer_Target` button on coarse pointers is promoted to the 44 px hit area (R-TOUCH) regardless of visual size. Toolbar button rows wrap to a new line below `md` rather than horizontally scrolling. +- **State contract:** + - **default:** variant fill/border per *Buttons* above; label meets R-CONTRAST (`{colors.on-filled}` on primary, `{colors.text}` on ghost/outline). + - **hover:** primary adds `{shadows.glow}` + `bg → {colors.accent-dark}`; ghost → `{colors.bg-card}`; outline swaps border + text to `{colors.accent}`. + - **focus:** per R-FOCUS. + - **active (press):** scales to 0.98; ghost → `{colors.bg-card2}`. + - **disabled:** `opacity: 0.5` + `cursor: not-allowed`; no hover/glow; still announced to AT. + - **error:** *n/a* — buttons carry no intrinsic error state (submit errors surface on fields). + +#### `{components.card}` / `{components.card-hover}` / `{components.row-card}` — cards & rows + +- **Responsive contract:** full-width within its band at every breakpoint; interior text follows R-WRAP (no truncation). Card interior padding is fixed 20 px at all breakpoints (page breathing comes from the 1600 px container, not padding expansion). +- **State contract:** + - **default:** `{colors.bg-card}`, 1-px `{colors.border}`, `{shadows.sm}` (L2). + - **hover (clickable only):** `{components.card-hover}` lifts −2 px, shadow → `{shadows.lg}` (L4), border → `{colors.border-strong}`; `{components.row-card}` only tints border → `{colors.border-md}` (no transform, kept calm for dense lists). + - **focus (clickable only):** per R-FOCUS. + - **active (clickable only):** border holds at `{colors.border-strong}`; no additional transform. + - **disabled:** *n/a* — non-interactive cards have no disabled state; a disabled clickable card falls back to the button disabled contract. + - **error:** *n/a* — error is expressed by the content inside (e.g. `{components.empty-state}` load-error), not the card shell. + +#### KPI strip / `{components.kpi-strip}` — page-opening counters + +- **Responsive contract:** + - **Mobile < 640 & Small 640–767:** 2-up grid (`grid-cols-2`). + - **Tablet 768–1023:** 2-up grid retained; cells full-width of their column. + - **Desktop 1024–1279 & Wide ≥ 1280:** 4-up grid (`lg:grid-cols-4`). Each cell keeps 20-px padding at every breakpoint; the value uses `tabular-nums` and truncates with the full text carried in `title` rather than reflowing the strip. +- **State contract:** + - **default:** `{components.card}` chrome around hairline-divided cells; optional 32 × 32 accent icon tile per cell. + - **hover:** interactive cells tint to `{colors.bg-card2}`; the strip does not lift or transform. + - **focus:** per R-FOCUS when the cell is interactive; *n/a* for display-only cells. + - **active:** *n/a* unless linked, then follows the clickable-card active contract. + - **disabled:** *n/a*. + - **error:** *n/a* — a metric with no value renders the missing-value placeholder inside the cell (see [Metric Display Contract](#metrics)), not a cell error state. + +#### `{components.tabs}` — pill-container tabs + +- **Responsive contract:** segmented pill row at `md+`; below `md` the row wraps (or scrolls its own container, never the page) per R-WRAP so every tab label stays legible. Tab labels never truncate. +- **State contract:** + - **default (inactive):** `{colors.bg-card}` fill, `{colors.text-muted}` label; `tabindex=-1` under roving-tabindex. + - **hover:** fill → `{colors.bg-card2}`. + - **focus:** the single active tab holds `tabindex=0`; arrow keys move focus with wrap-around; per R-FOCUS. + - **active (selected):** `{colors.accent}` fill + `{colors.on-filled}` text + soft glow `0 0 12 px rgba(129,109,248,0.2)`; exactly one tab has `aria-selected=true` and one visible `tabpanel`. + - **disabled:** disabled tab renders at `opacity: 0.5`, `aria-disabled=true`, skipped by roving navigation. + - **error (orphan item):** a tab with no matching `tabpanel` reference is **not rendered** and an error is surfaced. + +#### `{components.table}` — sortable data table + +- **Responsive contract:** + - **Mobile < 640 & Small 640–767:** the Evaluation-History surface presents the **card layout** (field-per-line, R-WRAP, no truncation); a raw table, where shown, is wrapped in `overflow-x-auto` so all columns are preserved (horizontal scroll inside the card shell, never at the page level). + - **Tablet 768–1023:** table with `overflow-x-auto`; all columns retained rather than dropped. + - **Desktop 1024–1279 & Wide ≥ 1280:** full table with the fixed, ordered columns `model · dataset · time · samples · score · status`. Header text uses `{typography.table-xs}` at 12 px and is never shrunk below the floor. +- **State contract:** + - **default:** header `{typography.table-xs}` UPPERCASE `{colors.text-muted}`; 1-px `{colors.border}` row dividers. + - **hover:** sortable header lifts to `{colors.text}`; clickable row fills `{colors.bg-card2}`. + - **focus:** sortable header / row focusable, per R-FOCUS. + - **active (sorted column):** header turns `{colors.accent}` with a `ChevronUp/Down` direction indicator. + - **disabled:** *n/a* — columns are always sortable/among the fixed set. + - **error / empty:** centered dimmed "No data" cell (or the `{components.empty-state}` surface when the whole view is empty). + +#### `{components.input}` / `{components.select}` / `Field_Primitive` — inputs & forms + +- **Responsive contract:** label/value pairs are two-column at `md+` (`grid-cols-1 md:grid-cols-2`) and single-column below `md`; label and helper text follow R-WRAP. Inputs are full-width of their column at every breakpoint. +- **State contract:** + - **default:** `{colors.bg-deep}` well, 1-px `{colors.border}`, `{colors.text}` value, `{colors.text-dim}` placeholder; a non-empty programmatically-associated label (`{colors.text-muted}`, UPPERCASE `{typography.label-xs}`). + - **hover:** border → `{colors.border-md}`. + - **focus:** border → `{colors.accent}` + 1-px `{colors.accent-dim}` ring (soft halo); per R-FOCUS. + - **active:** same as focus while editing. + - **disabled:** `opacity: 0.5`, `cursor: not-allowed`, `aria-disabled`; still exposes its accessible name. + - **error:** border + ring swap to the danger family, `aria-invalid=true`, `aria-describedby` points at a 12 px `{colors.danger}` helper line; the message is announced via a polite live region within 1 s and receives focus first among invalid fields on submit. + +#### `{components.empty-state}` — actionable empty states + +- **Responsive contract:** vertically-centered stack, full-width of its band at every breakpoint; the 64 × 64 icon tile and the two message lines wrap per R-WRAP and never truncate. Recovery-action buttons stack below `md` and sit inline at `md+`. +- **State contract:** + - **default:** icon tile (`{colors.accent}` welcome / `{colors.text-dim}` empty) + title + hint; renders within 300 ms of a completed zero-record load, with reason-specific text for `no-data` / `load-error` / `no-match`. + - **hover / focus / active:** carried by the 1–3 embedded recovery-action buttons per the `{components.button}` contract (each a `Pointer_Target` ≥ 44 × 44 on touch, R-TOUCH); every action has a non-empty `navigateTo`. + - **disabled:** *n/a* — recovery actions are always actionable. + - **error:** the `load-error` reason **is** this component's error presentation (distinct copy + retry action); it does not render a blank region. + +#### `{components.eval-run-card}` — eval timeline run row + +- **Responsive contract:** single full-width column at every breakpoint (the timeline is already a vertical list). At 390 px all metadata fields (model, dataset, time, samples, score, status, per-benchmark score chips) stay visible and wrap per R-WRAP — no ellipsis, no horizontal scroll. The score-chip row wraps onto multiple lines rather than clipping. +- **State contract:** + - **default:** `{colors.bg-card}`, 1-px `{colors.border}`; bold model name + score badge; mono timestamp + wrapped `{components.score-chip}` set. + - **hover:** border tint → `{colors.border-md}` only (no lift, calm for dense lists). + - **focus:** per R-FOCUS (the row is a button). + - **active (press):** border holds at `{colors.border-md}`; navigates to detail. + - **disabled:** *n/a*. + - **error:** *n/a* — a run with missing values shows placeholders/`N/A` per the metric contract, not a card error state. + +#### `{components.score-chip}` / `{components.badge}` / `{components.filter-chip}` — chips & pills + +- **Responsive contract:** chips are meant to **wrap** — a chip row flows onto multiple lines with an 8-px gap at every breakpoint and never truncates or horizontally scrolls (R-WRAP). Chip content (`"benchmark-name 87.3"`) is `tabular-nums` mono and stays whole. +- **State contract:** + - **default:** `{components.score-chip}` outline (transparent bg + 1-px `scoreColor` border + `scoreColor` text); `{components.badge}` uses an 8–10 % alpha bg + saturated fg. Both meet R-CONTRAST for the essential value. + - **hover:** `{components.filter-chip}` reveals its dismiss control; score/badge pills are static. + - **focus:** `{components.filter-chip}` dismiss control and any interactive chip follow R-FOCUS and R-TOUCH (≥ 44 × 44 hit area on touch). + - **active:** dismiss removes the chip; static chips have no active state. + - **disabled:** *n/a*. + - **error:** *n/a*. + +#### `ex-compare-column` / `{components.card}` (compare) — compare-view model column + +- **Responsive contract:** + - **Mobile < 640 & Small 640–767:** columns stack vertically (one model per full-width block); display label and metric rows wrap per R-WRAP. + - **Tablet 768–1023:** 2 columns side-by-side. + - **Desktop 1024–1279 & Wide ≥ 1280:** 2–3 columns side-by-side; beyond 3 models the view collapses into a numbered legend rather than inventing a 4th slot color. +- **State contract:** + - **default:** header carries the model+dataset display label and one of the `{compare.0..2}` slot accents (`dot` / `border` / `bg-header`). + - **hover:** column header/row tint strengthens via the slot's `bg-hl` where interactive. + - **focus:** compare-selection controls follow R-FOCUS and R-TOUCH. + - **active (selected for compare):** slot accent applied; selection persists across sort/filter and is reflected in the sticky selection tray count. + - **disabled (incompatible):** score-comparison selection is unbounded; prediction comparison accepts up to `MAX_COMPARE_SLOTS = 3` runs. An incompatible run shows its incompatibility reason and retains existing selection. + - **error:** incompatibility reason text is the error presentation; it does not drop the selection. + ### Examples (illustrative) > These `ex-*` surfaces mirror the brand-native primitives for downstream consumers (kits, mockups, Stitch generation). Each references existing components so a re-skin re-skins all surfaces consistently. -**`ex-metric-tile`** — Dashboard KPI tile. Re-uses `{components.kpi-card}` with gradient icon tile + tabular value + uppercase label. -- Properties: `backgroundColor`, `iconGradient`, `rounded`, `padding`, `valueTypography`, `labelTypography`. +**`ex-metric-tile`** — Dashboard KPI cell. Re-uses one cell of `{components.kpi-strip}` with accent icon tile + tabular value + label. +- Properties: `backgroundColor`, `iconTint`, `rounded`, `padding`, `valueTypography`, `labelTypography`. **`ex-eval-run-row`** — A row in the eval timeline. Re-uses `{components.eval-run-card}` chrome. - Properties: `backgroundColor`, `borderColor`, `rounded`, `padding`, `scoreColor`, `chipColor`. @@ -649,9 +800,55 @@ Disabled = `opacity: 0.5` + `cursor: not-allowed`. Transitions use `{tokens.tran **`ex-compare-column`** — A model column in the compare view, using one of the three `{compare.0..2}` slot colors. - Properties: `dotColor`, `borderColor`, `headerBackground`. -## Do's and Don'ts +## Metric Display Contract {#metrics} -### Do +> **Addressable section — `metrics`.** How raw metric values become on-screen text. Display form is decided by a metric's `MetricDisplaySpec` metadata — never by inspecting the numeric magnitude — so the same value renders consistently across list, detail, compare, and export surfaces. The contract below is implemented by `src/domain/metric/MetricDisplaySpec.ts` and `src/domain/metric/metricFormat.ts`. + +### Display Rules + +- **Spec-driven, not value-driven.** `resolveSpec(key, registry)` returns the metric's spec (or `DEFAULT_METRIC_SPEC` with `isFallback = true`). The spec's `kind` — not the value — selects the presentation. A raw value greater than `1` never triggers percentage formatting. +- **Bounded ratio → percentage.** A `bounded-ratio` metric (domain `0–1`) renders as a percentage to `percentPrecision` decimals using round-half-up, and preserves a 4-decimal raw value alongside it. `bounded` metrics (domain `0–100`) display on their native scale. +- **Unbounded / native → raw with unit.** Unbounded or native-unit metrics keep their unit and render at `rawPrecision`; they are never converted to a percentage. +- **Missing vs. legitimate zero.** A missing value renders `MISSING_PLACEHOLDER` and sets `isMissing = true`; a real `0` renders as a formatted zero. The two are always distinguishable. +- **Undefined spec fallback.** When no spec is registered for a key, the value renders as a 4-decimal raw number with `isSpecUndefined = true`, and the implementation-level metric name is hidden behind a localized `labelKey`. +- **Rounding.** All decimal rounding uses `roundHalfUp(value, precision)` so `.5` cases round up deterministically across every surface. + +### Threshold Semantics + +- **Prediction threshold is a view-only filter.** A per-view threshold only annotates or filters rows (`Below filter` / `Above filter`); it never participates in pass/fail and never affects other views. +- **Native outcome is independent.** A benchmark's native `outcome` (pass/fail) is derived and displayed independently of any prediction threshold. + +## Decision Records {#decisions} + +> **Addressable section — `decisions`.** The load-bearing decisions behind this system, captured so future changes understand what is intentional versus incidental. Each decision is normative; the *Do's and Don'ts* that follow are the enforceable expression of these decisions. + +### D1 — Dual-theme parity, not translation + +Dark and light are two equally weighted voices of one brand, not one canonical mode re-tinted. They share vocabulary (type, spacing, radii, components) but use *opposite materials* to produce the same hierarchy: dark uses translucent-violet hairlines on near-black; light uses solid warm-grey hairlines on cream. Every new color token must define both a dark and a light value in the same commit. + +### D2 — Solid-hex hairlines on the light theme + +Light-theme borders are solid warm-grey hex (`#e6dfd8` / `#d6cdbe` / `#c1b6a3`), never translucent violet. Violet at any plausible alpha composites to near-invisible against a white card on cream. This is the single most-broken light-theme rule; see [Design Tokens → Hairlines](#tokens). + +### D3 — Dynamic HSL score gradient is a brand asset + +Scores map to `hsl(score × 120, 70%, 45%)` (red → yellow → green), computed inline and never stored as a static palette. Per-theme legibility is tuned only through CSS vars (`--score-fg-s`, `--score-fg-l`, `--score-bg-a-mul`); the formula itself is not re-skinned into buckets and gains no fourth hue. + +### D4 — System font stack, no web font + +The brand loads no `@font-face`; cross-platform `system-ui` / `ui-monospace` stacks resolve to each OS's native face. This is deliberate — it reads as "native developer tool," and the trade-off is per-OS rendering variance. + +### D5 — Token single source of truth and drift governance + +Token values live once in `evalscope/web/src/index.css`. Run `npm run design:tokens` after changing shared runtime tokens; `scripts/drift/tokenDrift.ts` verifies that this document's frontmatter was generated from the current CSS values. + +### D6 — Additive, reversible migration + +Refactors preserve raw values, keep chart data-table fallbacks, and retain old components until visual-regression parity is proven. Web API response contracts are not duplicated during migration: backend Pydantic models are the single source of truth, and the frontend consumes generated TypeScript types. + +### Do's and Don'ts + +#### Do - Reserve `{colors.accent}` (`#816DF8`) for primary CTAs, active states, focus rings, and the wordmark accent. Brand violet IS the conversion target — keep it under ~10 % of any screen. - Use `{rounded.sm}` 8 px for buttons / inputs / tabs, `{rounded.md}` 12 px for cards, `{rounded.full}` only for badges / chips / score pills. Each shape signals its category. @@ -664,19 +861,19 @@ Disabled = `opacity: 0.5` + `cursor: not-allowed`. Transitions use `{tokens.tran - Animate page transitions with `fadeInUp` (12 px translate + opacity, 400 ms ease-out) and stagger children at 60 ms. The motion is subtle — don't lengthen it. - Persist the `data-theme` to `localStorage` and apply it pre-paint in `index.html` to avoid FOUC. Tokens are theme-agnostic by name; values flip. -### Don't +#### Don't - Don't introduce a 6th brand hue or a 4th compare-slot color. The palette is closed at violet + emerald + amber + red + slate (plus the dynamic HSL score). New accents flatten the voice. -- Don't render headlines in all-caps. UPPERCASE is the eyebrow voice (12 px / 10 px micro-labels) — never the title voice. Card titles and model names stay sentence-case. +- Don't render headlines in all-caps. UPPERCASE is the 12 px eyebrow/table-label voice — never the title voice. Card titles and model names stay sentence-case. - Don't promote the sans to `font-extrabold` / `font-black`. The display weight ceiling is **700**. -- Don't use `{colors.text-dim}` for essential UI text on either theme — its ~3.6 : 1 contrast against `{colors.bg-card}` clears AA Large (3 : 1) but is below WCAG AA Normal (4.5 : 1). Reserve for ≥ 14 px non-essential metadata (timestamps, "empty" labels, scrollbar thumb). Every code-side use must carry the inline note `// text-dim allowed: non-essential ≥14px metadata (DESIGN.md §Text)` so reviewers can audit it. +- Don't use `{colors.text-dim}` for essential UI text on either theme — its ~3.6 : 1 contrast against `{colors.bg-card}` clears AA Large (3 : 1) but is below WCAG AA Normal (4.5 : 1). Reserve textual use for ≥ 14 px non-essential metadata (timestamps and empty labels). Placeholder text, decorative icons, separators, disabled controls, and scrollbar thumbs are exempt because they are not essential readable content. Add the inline note `// text-dim allowed: non-essential ≥14px metadata (DESIGN.md §Text)` when a textual use is not self-evidently a placeholder or empty-state annotation; do not scatter the note onto every decorative icon. - Don't reuse a dark-theme shadow value verbatim on light. The light palette stacks two **warm-ink-tinted** drops (`rgba(20,20,19,0.04)` + `rgba(20,20,19,0.06)`) — a single `rgba(0,0,0,0.07)` drop on cream reads as a page smudge, not as a lifted card. Do not slate-tint the light shadows either (`rgba(15,23,42,*)`) — slate on cream reads as a cool-grey smear that fights the warm canvas. See *Elevation & Depth*. - **Don't use translucent violet for light-theme hairlines.** This is the most-broken light-theme rule. The light `{colors.border-light}` is a SOLID warm-grey hex (`#e6dfd8`) — translucent violet at *any* plausible alpha (0.10 / 0.20 / 0.30 / 0.40) composites to near-invisible against a white card on cream and leaves every card boundary undefined. Outline buttons and input rings will vanish. The dark theme uses translucent violet because the near-black bg-to-card luminance delta carries the boundary; the light theme has no such delta and must use a concrete material. - Don't introduce a cool-grey or pure-white surface to the light theme. The light palette is warm-cream by design (`#faf9f5` canvas, `#f0ebe1` deep, `#f5f0e7` elevated, `#ffffff` cards). A cool-grey `#f5f6fa` or `#eef0f7` band breaks the warm-coherent rhythm and reverts the system to "any other AI dashboard." - Don't drop a single heavy 8-px-blur drop-shadow on a card. The dark theme requires *deeper* multi-stop shadows (`rgba(0,0,0,0.55)` at 20-40 px) — soft drops disappear on near-black. - Don't apply `{gradients.brand}` to body text or table cells. Gradient-text is for hero / wordmark moments only. -- Don't bypass `{components.button}` to write custom `bg-[var(--accent)]` buttons inline. The button variants encode the glow, the scale-press, and the disabled state — re-deriving them by hand drifts the brand. -- Don't use inline `style={{ background: 'var(--xxx)' }}` when a Tailwind class or the `formStyles` helper would do. Inline styles bypass the token abstraction and break theme switching for the hover state. +- Standalone actions and conversion CTAs use `{components.button}` variants; do not re-derive their glow, press, focus, or disabled behavior with a one-off `bg-[var(--accent)]`. Component-internal controls with a distinct semantic contract (tabs, segmented selectors, list rows, score chips) may compose token classes directly, but their states belong in that component's contract and tests. +- Prefer token-backed Tailwind classes or `formStyles` for static surfaces. Inline styles are reserved for values that are genuinely computed at runtime (for example score colors, chart series colors, progress widths, and measured geometry); they must still resolve through documented tokens/formulas where applicable. - Don't pair `{rounded.full}` pill shapes with `{rounded.md}` cards as siblings inside the same control group — pills are for *data*, sm/md radii are for *interactive containers*. Mixing them on the same row breaks the shape grammar. - Don't ignore `prefers-color-scheme` on first visit. If the user has never toggled, fall back to the OS preference before defaulting to dark. - Don't widen `{spacing.xl}` (20 px) section gaps past `{spacing.2xl}` (24 px). The product is information-dense by design; extra whitespace makes the dashboard feel half-empty rather than airy. diff --git a/evalscope/MANIFEST.in b/evalscope/MANIFEST.in index ca2a7f2..9210c38 100644 --- a/evalscope/MANIFEST.in +++ b/evalscope/MANIFEST.in @@ -4,16 +4,23 @@ include README.md recursive-include evalscope * # Exclude cache/compiled artifacts -global-exclude *.py[cod] __pycache__ *.so *.dylib +global-exclude *.py[cod] __pycache__ *.so *.dylib .DS_Store # Exclude large data files global-exclude *.h5 *.hdf5 *.parquet *.bin *.safetensors *.gguf *.pth *.pt # Exclude frontend dev files (only dist/ is needed at runtime) +prune evalscope/web/coverage prune evalscope/web/node_modules +prune evalscope/web/public +prune evalscope/web/scripts prune evalscope/web/src +exclude evalscope/web/.gitignore +exclude evalscope/web/README.md +exclude evalscope/web/index.html exclude evalscope/web/package.json exclude evalscope/web/package-lock.json exclude evalscope/web/tsconfig*.json exclude evalscope/web/vite.config.ts +exclude evalscope/web/vitest.config.ts exclude evalscope/web/eslint.config.js diff --git a/evalscope/Makefile b/evalscope/Makefile index e3935bc..b32e39c 100644 --- a/evalscope/Makefile +++ b/evalscope/Makefile @@ -1,6 +1,9 @@ # default rule default: install +PYTHON ?= python +DIST_DIR ?= $(CURDIR)/dist + # ============================================================================ # Documentation Generation # ============================================================================ @@ -108,10 +111,40 @@ web-install: web-build: cd evalscope/web && npm install && npm run build +.PHONY: web-contracts-check +web-contracts-check: + cd evalscope/web && npm run contracts:check + +.PHONY: web-release-build +web-release-build: + cd evalscope/web && npm ci + $(MAKE) web-contracts-check + cd evalscope/web && npm run build + .PHONY: web-dev web-dev: cd evalscope/web && npm install && npm run dev +# ============================================================================ +# Release Package +# ============================================================================ + +.PHONY: package +package: + $(MAKE) web-release-build + $(MAKE) package-build + $(MAKE) package-check + +# Run outside the repository so a local build/ directory cannot shadow the Python build package. +.PHONY: package-build +package-build: + cd "$(CURDIR)/.." && $(PYTHON) -m build "$(CURDIR)" --outdir "$(DIST_DIR)" + +.PHONY: package-check +package-check: + $(PYTHON) scripts/release/verify_package.py --dist-dir "$(DIST_DIR)" + $(PYTHON) -m twine check "$(DIST_DIR)"/* + # ============================================================================ # Development # ============================================================================ @@ -123,7 +156,6 @@ lint: .PHONY: dev dev: pip install -e '.[dev,perf,docs]' - pip install pre-commit .PHONY: install install: diff --git a/evalscope/README.md b/evalscope/README.md index 63e6fda..5e84503 100644 --- a/evalscope/README.md +++ b/evalscope/README.md @@ -1,176 +1,394 @@ -# EvalScope 评测仓库 +

+
+ +
+

-## 0. Benchmarks +

+ 中文   ｜   English   +

-### 常用 benchmark 分类 +

+ +PyPI version +PyPI - Downloads + +GitHub stars +Documentation Status +

-```python -benchmark_categories = { - "代码与工程": [ - "terminal_bench_v2", - "live_code_bench", - "swe_bench_multilingual_agentic", - "swe_bench_pro", - "swe_bench_verified", - "bigcodebench", - "humaneval", - ], - "推理与数学": [ - "gpqa_diamond", - "hle", - "aime24", - "aime25", - "aime26", - "hmmt26", - "imo_answerbench", - "super_gpqa", - "gsm8k", - "competition_math", - ], - "智能体与工具": [ - "browsecomp", - "mcp_atlas", - "tau2_bench", - ], - "知识与语言理解": [ - "mmlu_pro", - "simple_qa", - "arc", - "bbh", - "cmmlu", - "drop", - "hellaswag", - "mmlu", - "trivia_qa", - "winogrande", - ], - "长上下文": [ - "longbench_v2", - "openai_mrcr", - ], -} -``` +

+ 📖 中文文档   ｜   📖 English Documentation +

-### 分组运行 -```python -group_1 = [ - 'terminal_bench_v2', - 'gpqa_diamond', - 'hle', - 'aime24', - 'aime25', - 'mmlu_pro', - 'simple_qa', - 'arc', - 'bbh', - 'browsecomp', -] +> ⭐ If you like this project, please click the "Star" button in the upper right corner to support us. Your support is our motivation to move forward! -group_2 = [ - 'live_code_bench', - 'swe_bench_pro', - 'aime26', - 'hmmt26', - 'imo_answerbench', - 'super_gpqa', - 'drop', - 'hellaswag', - 'mmlu', - 'openai_mrcr', - 'mcp_atlas', -] +## 📝 Introduction -group_3 = [ - 'swe_bench_multilingual_agentic', - 'swe_bench_verified', - 'bigcodebench', - 'humaneval', - 'gsm8k', - 'competition_math', - 'cmmlu', - 'trivia_qa', - 'winogrande', - 'longbench_v2', - 'tau2_bench', -] -``` - -## 1. 安装 +EvalScope is a one-stop LLM evaluation framework built by the [ModelScope Community](https://modelscope.cn/). Just one command to start — it supports model capability evaluation, inference performance stress testing, and result visualization. ```bash -conda create -n evalscope python==3.12 -y -git clone https://github.com/modelscope/evalscope -pip install -e . -pip install 'evalscope[terminal_bench,swe_bench,openai_mrcr]' \ - 'git+https://github.com/sierra-research/tau2-bench@v0.2.0' -pip install 'evalscope[sandbox]' -pip install 'swebench==4.1.0' +pip install evalscope +evalscope eval --model your-model-name --api-url $OPENAI_API_BASE_URL --api-key $OPENAI_API_KEY --eval-type openai_api --datasets gsm8k --limit 5 ``` -## 2. 运行命令 +## ✨ Key Features + +- **📚 Comprehensive Evaluation Benchmarks**: Built-in multiple industry-recognized evaluation benchmarks including MMLU, C-Eval, GSM8K, and more. +- **🧩 Multi-modal and Multi-domain Support**: Supports evaluation of various model types including Large Language Models (LLM), Vision Language Models (VLM), Embedding, Reranker, AIGC, and more. +- **🚀 Multi-backend Integration**: Seamlessly integrates multiple evaluation backends including OpenCompass, VLMEvalKit, RAGEval to meet different evaluation needs. +- **🤖 Agent Evaluation Mode**: Drives benchmarks (e.g. GSM8K, AIME, SWE-bench Agentic) inside a controlled multi-turn AgentLoop with pluggable strategies, tools and Docker sandbox; full per-sample Agent Trace is recorded and visualizable. +- **⚡ Inference Performance Testing**: Provides powerful model service stress testing tools, supporting multiple performance metrics such as TTFT, TPOT. +- **📊 Interactive Reports**: Provides a Web Dashboard for multi-dimensional model comparison, report overview and detailed inspection. +- **⚔️ Arena Mode**: Supports multi-model battles (Pairwise Battle), intuitively ranking and evaluating models. +- **🔧 Highly Extensible**: Developers can easily add custom datasets, models and evaluation metrics. + +## 📊 Visualization Preview + +EvalScope provides an interactive Web Dashboard for multi-dimensional model comparison and in-depth analysis. + + + + + + + + + + +
+ Dashboard +

Dashboard Overview

+ 		+ Model Compare +

Model Comparison

+
+ Report Overview +

Report Overview

+ 		+ Report Predictions +

Prediction Details

+
+ +For details, please refer to [📖 Visualizing Evaluation Results](https://evalscope.readthedocs.io/en/latest/get_started/visualization.html). + +## 🎉 What's New + +- 🔥 **[2026.08.24] v1.11.0** Introduced published evaluation versions for reproducible benchmark results; improved report semantics and incomplete-run handling; strengthened multimodal media loading and task-config validation. +- 🔥 **[2026.08.13]** Improved evaluation reports with unified metric semantics and more reliable Agent Trace step grouping and tool-call/result linking. +- 🔥 **[2026.08.10]** Added **AutomationBench**, **JobBench**, **MiniWoB**, **OmniDocBench-v1.6**, **PerceptionBench**, **ScreenSpot-Pro**, **PLawBench**, **PMC-VQA**, **HiPhO**, **LogicVista**, and **CC-OCR-V2** benchmarks. +- 🔥 **[2026.07.21]** Added **Claw-Eval**, **ResearchRubrics**, **Toolathlon** (agent), **TVBench** (video), **WideSearch**, and **PerspectiveGap** benchmarks. +- 🔥 **[2026.07.03]** Added **CharXiv** & **BabyVision** (chart understanding, visual cognition) and **ERQA** & **WorldVQA** (entity-recognition QA with LLM-judge + CoT) multimodal benchmarks. +- 🔥 **[2026.06.23]** Major agent & code evaluation expansion: added **BigCodeBench**, **SWE-bench Multilingual**, **BrowseComp**, **MCP-Atlas**, **GDPval** benchmarks; added **OpenCode** / **OpenHands** runners; refactored adapter architecture with `AudioLanguageAdapter`, unified `FunctionCallAdapter`, and public `run_agent_loop` API. +- 🔥 **[2026.06.16]** Added full-reference **image quality metrics** (SSIM, PSNR, etc.), long-context benchmarks (**LoCoMo QA**, **LongMemEval**), **Caption** & **Maritime-OCR-Bench** benchmarks; perf module now supports unified `--data-source` and parallelized request generation. +- 🔥 **[2026.06.02]** Refactored **RAG evaluation** module: upgraded to MTEB 2.x and RAGAS 0.4.x, with unified Pydantic-based configs. See the [RAGEval guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/index.html). +- 🔥 **[2026.05.27]** Added **Trie agentic trace replay** for perf benchmarking: three new dataset plugins (`trie_agentic_coding` / `trie_code_qa` / `trie_office_work`) replay real multi-turn agent traces with per-turn token caps and tool-call latency simulation. Also introduced a `--duration` wall-clock budget for all benchmark modes and a `Turn` dataclass for per-turn overrides. +- 🔥 **[2026.05.27]** Added **Vendor Verifier benchmarks** (`k2_verifier`, `kimi_verifier`, `minimax_verifier`) for validating whether third-party API deployments faithfully reproduce official model behavior, with a shared `FunctionCallAdapter` base class. +- 🔥 **[2026.05.26]** Added the [GAIA](https://evalscope.readthedocs.io/en/latest/third_party/gaia.html) agent benchmark (multi-turn ReAct + `bash` in a Docker sandbox, official rule-based scorer) and generic [MCP server](https://evalscope.readthedocs.io/en/latest/user_guides/agent/native.html#mcp-server-tools) support — any `NativeAgentConfig`-driven benchmark can now plug in stdio / HTTP / SSE MCP servers (`fetch`, web search, GitHub, ...) without per-benchmark wiring. +- 🔥 **[2026.05.22]** Introduced the **External Agent Bridge** mode: evaluate off-the-shelf agent CLIs such as Anthropic's [Claude Code](https://github.com/anthropics/claude-code) and OpenAI's [Codex](https://github.com/openai/codex) directly through EvalScope. The bridge transparently forwards each CLI's LLM traffic (Anthropic Messages / OpenAI Chat / OpenAI Responses, including SSE streaming) to your evaluation model, while recording the full trajectory as an `agent_trace`. Bring-your-own-runner via `@register_runner`. See the [External Agent Bridge guide](https://evalscope.readthedocs.io/en/latest/user_guides/agent/bridge.html). +- 🔥 **[2026.05.19]** Added support for [SWE-bench_Pro](https://evalscope.readthedocs.io/en/latest/third_party/swe_bench_pro.html) and [τ³-bench](https://evalscope.readthedocs.io/en/latest/third_party/tau3_bench.html): SWE-bench_Pro is a more challenging multilingual long-horizon software-engineering benchmark from Scale AI (recommended over the original SWE-bench for less data contamination and broader language coverage; per-instance Docker images are pulled directly from DockerHub, no local image build required); τ³-bench is the v1.0.0 release of the tau-bench family, extending τ²-bench with a new `banking_knowledge` retrieval domain (RAG), 75+ task fixes across existing domains, and pluggable retrieval pipelines (BM25 / embeddings / rerankers / sandbox shell). + +

More historical updates + +- 🔥 **[2026.05.15]** Introduced **Agent Evaluation Mode**: any benchmark based on `DefaultDataAdapter` (GSM8K, AIME, IFEval, etc.) can now be driven through a multi-turn AgentLoop with pluggable strategies (`function_calling` / `react` / `swe_bench_*`), tools (`bash` / `python_exec` / `submit`) and `local` / `docker` environments. Per-sample `agent_trace` is recorded and rendered step-by-step in the dashboard's Predictions tab. See the [Agent Evaluation guide](https://evalscope.readthedocs.io/en/latest/user_guides/agent/native.html) for details. +- 🔥 **[2026.05.08]** Partnered with [LightSeek](https://lightseek.org/) to launch [TokenSpeed](https://lightseek.org/blog/lightseek-tokenspeed.html), a speed-of-light LLM inference engine for agentic workloads. EvalScope provides the SWE-smith benchmarking pipeline — using real coding-agent traces to measure per-GPU throughput (TPM) and per-user latency (TPS) — serving as the official benchmark tool for TokenSpeed performance evaluation. Refer to the [SWE-smith usage guide](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test/multi_turn.html#swe-smith) to get started. + +- 🔥 **[2026.05.07]** Replaced the Gradio-based WebUI with a new React + Vite web interface for better performance and user experience. +- 🔥 **[2026.04.23]** Added support for recording performance (perf) metrics during evaluation tasks, enabling simultaneous tracking of model accuracy and inference efficiency metrics such as TTFT, TPOT, and throughput in a single evaluation run. +- 🔥 **[2026.04.17]** Added support for multi-turn conversation performance stress testing, enabling load testing of dialogue-based model services with multi-turn context. Refer to the [usage documentation](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test/examples.html). +- 🔥 **[2026.04.10]** Added support for [TIR-Bench](https://arxiv.org/abs/2511.01833) (Thinking-with-Images Reasoning Benchmark), a multimodal benchmark evaluating agentic visual reasoning capabilities of vision-language models. +- 🔥 **[2026.03.24]** Added support for Agent Skill. Any agent model that supports Skill/Tool calling can use natural language to drive EvalScope for model evaluation, performance benchmarking, and result visualization. +- 🔥 **[2026.03.09]** Added support for evaluation progress tracking and HTML format visualization report generation. +- 🔥 **[2026.03.02]** Added support for Anthropic Claude API evaluation. Use `--eval-type anthropic_api` to evaluate models via Anthropic API service. +- 🔥 **[2026.02.03]** Comprehensive update to dataset documentation, adding data statistics, data samples, usage instructions and more. +- 🔥 **[2026.01.13]** Added support for Embedding and Rerank model service stress testing. +- 🔥 **[2025.12.26]** Added support for Terminal-Bench-2.0, which evaluates AI Agent performance on 89 real-world multi-step terminal tasks. +- 🔥 **[2025.12.18]** Added support for SLA auto-tuning model API services. +- 🔥 **[2025.12.16]** Added support for audio evaluation benchmarks such as Fleurs, LibriSpeech; added support for multilingual code evaluation benchmarks such as MultiplE, MBPP. +- 🔥 **[2025.12.02]** Added support for custom multimodal VQA evaluation; added support for visualizing model service stress testing in ClearML. +- 🔥 **[2025.11.26]** Added support for OpenAI-MRCR, GSM8K-V, MGSM, MicroVQA, IFBench, SciCode benchmarks. +- 🔥 **[2025.11.18]** Added support for custom Function-Call (tool invocation) datasets to test whether models can timely and correctly call tools. +- 🔥 **[2025.11.14]** Added support for SWE-bench_Verified, SWE-bench_Lite, SWE-bench_Verified_mini code evaluation benchmarks. +- 🔥 **[2025.11.12]** Added `pass@k`, `vote@k`, `pass^k` and other metric aggregation methods; added support for multimodal evaluation benchmarks such as A_OKVQA, CMMU, ScienceQA, V*Bench. +- 🔥 **[2025.11.07]** Added support for τ²-bench, an extended and enhanced version of τ-bench that includes a series of code fixes and adds telecom domain troubleshooting scenarios. +- 🔥 **[2025.10.30]** Added support for BFCL-v4, enabling evaluation of agent capabilities including web search and long-term memory. +- 🔥 **[2025.10.27]** Added support for LogiQA, HaluEval, MathQA, MRI-QA, PIQA, QASC, CommonsenseQA and other evaluation benchmarks. Thanks to @[penguinwang96825](https://github.com/penguinwang96825) for the code implementation. +- 🔥 **[2025.10.26]** Added support for Conll-2003, CrossNER, Copious, GeniaNER, HarveyNER, MIT-Movie-Trivia, MIT-Restaurant, OntoNotes5, WNUT2017 and other Named Entity Recognition evaluation benchmarks. Thanks to @[penguinwang96825](https://github.com/penguinwang96825) for the code implementation. +- 🔥 **[2025.10.21]** Optimized sandbox environment usage in code evaluation, supporting both local and remote operation modes. +- 🔥 **[2025.10.20]** Added support for evaluation benchmarks including PolyMath, SimpleVQA, MathVerse, MathVision, AA-LCR; optimized evalscope perf performance to align with vLLM Bench. +- 🔥 **[2025.10.14]** Added support for OCRBench, OCRBench-v2, DocVQA, InfoVQA, ChartQA, and BLINK multimodal image-text evaluation benchmarks. +- 🔥 **[2025.09.22]** Code evaluation benchmarks (HumanEval, LiveCodeBench) now support running in a sandbox environment. +- 🔥 **[2025.09.19]** Added support for multimodal image-text evaluation benchmarks including RealWorldQA, AI2D, MMStar, MMBench, and OmniBench, as well as pure text evaluation benchmarks such as Multi-IF, HealthBench, and AMC. +- 🔥 **[2025.09.05]** Added support for vision-language multimodal model evaluation tasks, such as MathVista and MMMU. +- 🔥 **[2025.09.04]** Added support for image editing task evaluation, including the [GEdit-Bench](https://modelscope.cn/datasets/stepfun-ai/GEdit-Bench) benchmark. +- 🔥 **[2025.08.22]** Version 1.0 Refactoring. Break changes, please [refer to](https://evalscope.readthedocs.io/en/latest/get_started/basic_usage.html#switching-to-version-v1-0). +- 🔥 **[2025.07.18]** The model stress testing now supports randomly generating image-text data for multimodal model evaluation. +- 🔥 **[2025.07.16]** Support for [τ-bench](https://github.com/sierra-research/tau-bench) has been added. +- 🔥 **[2025.07.14]** Support for "Humanity's Last Exam" ([Humanity's-Last-Exam](https://modelscope.cn/datasets/cais/hle)). +- 🔥 **[2025.07.03]** Refactored Arena Mode. +- 🔥 **[2025.06.28]** Optimized custom dataset evaluation; enhanced LLM judge usage. +- 🔥 **[2025.06.19]** Added support for the [BFCL-v3](https://modelscope.cn/datasets/AI-ModelScope/bfcl_v3) benchmark. +- 🔥 **[2025.06.02]** Added support for the Needle-in-a-Haystack test. +- 🔥 **[2025.05.29]** Added support for two long document evaluation benchmarks: DocMath and FRAMES. +- 🔥 **[2025.05.16]** Model service performance stress testing now supports setting various levels of concurrency. +- 🔥 **[2025.05.13]** Added support for the ToolBench-Static dataset, DROP and Winogrande benchmarks. +- 🔥 **[2025.04.29]** Added Qwen3 Evaluation Best Practices. +- 🔥 **[2025.04.27]** Support for text-to-image evaluation. +- 🔥 **[2025.04.10]** Model service stress testing tool now supports the `/v1/completions` endpoint. +- 🔥 **[2025.04.08]** Support for evaluating embedding model services compatible with the OpenAI API has been added. +- 🔥 **[2025.03.27]** Added support for AlpacaEval and ArenaHard evaluation benchmarks. +- 🔥 **[2025.03.20]** The model inference service stress testing now supports generating prompts of specified length using random values. +- 🔥 **[2025.03.13]** Added support for the LiveCodeBench code evaluation benchmark. +- 🔥 **[2025.03.11]** Added support for the SimpleQA and Chinese SimpleQA evaluation benchmarks. +- 🔥 **[2025.03.07]** Added support for the QwQ-32B model evaluation. +- 🔥 **[2025.03.04]** Added support for the SuperGPQA dataset. +- 🔥 **[2025.03.03]** Added support for evaluating the IQ and EQ of models. +- 🔥 **[2025.02.27]** Added support for evaluating the reasoning efficiency of models. +- 🔥 **[2025.02.25]** Added support for MuSR and ProcessBench benchmarks. +- 🔥 **[2025.02.18]** Supports the AIME25 dataset. +- 🔥 **[2025.02.13]** Added support for evaluating DeepSeek distilled models. +- 🔥 **[2025.01.20]** Support for visualizing evaluation results. +- 🔥 **[2025.01.07]** Native backend: Support for model API evaluation. +- 🔥🔥 **[2024.12.31]** Support for adding benchmark evaluations. +- 🔥 **[2024.12.13]** Model evaluation optimization. +- 🔥 **[2024.11.26]** The model inference service performance evaluator has been completely refactored. +- 🔥 **[2024.10.31]** The best practice for evaluating Multimodal-RAG has been updated. +- 🔥 **[2024.10.23]** Supports multimodal RAG evaluation. +- 🔥 **[2024.10.8]** Support for RAG evaluation. +- 🔥 **[2024.09.18]** Documentation added blog module. +- 🔥 **[2024.09.12]** Support for LongWriter evaluation. +- 🔥 **[2024.08.30]** Support for custom dataset evaluations. +- 🔥 **[2024.08.20]** Updated the official documentation. +- 🔥 **[2024.08.09]** Simplified the installation process. +- 🔥 **[2024.07.31]** Important change: The package name `llmuses` has been changed to `evalscope`. +- 🔥 **[2024.07.26]** Support for **VLMEvalKit** as a third-party evaluation framework. +- 🔥 **[2024.06.29]** Support for **OpenCompass** as a third-party evaluation framework. +- 🔥 **[2024.06.13]** EvalScope integrates with SWIFT; Integrated the Agent evaluation dataset ToolBench. + +
+ +## 🚀 Quick Start + +### Installation + +```shell +pip install evalscope +``` + +> For detailed installation instructions (source install, extra dependencies, etc.), please refer to the [📖 Installation Guide](https://evalscope.readthedocs.io/en/latest/get_started/installation.html). + +### Method 1. Evaluate an Online Model API (Recommended for beginners, no GPU required) + +Supports any OpenAI API-compatible model service. Just set `$OPENAI_API_BASE_URL` and `$OPENAI_API_KEY` and you are ready to go: + +```bash +evalscope eval \ + --model your-model-name \ + --api-url $OPENAI_API_BASE_URL \ + --api-key $OPENAI_API_KEY \ + --eval-type openai_api \ + --datasets gsm8k arc \ + --limit 5 +``` + +### Method 2. Evaluate a Local Model + +Evaluate a local model (auto-downloaded from ModelScope): + +```bash +evalscope eval \ + --model Qwen/Qwen2.5-0.5B-Instruct \ + --datasets gsm8k arc \ + --limit 5 +``` + +### Method 3. Using Python Code ```python from evalscope import run_task, TaskConfig -datasets = [ - 'live_code_bench', - 'swe_bench_pro', - 'aime26', - 'hmmt26', - 'imo_answerbench', - 'super_gpqa', - 'drop', - 'hellaswag', - 'mmlu', - 'openai_mrcr', - 'mcp_atlas', -] - task_cfg = TaskConfig( - collect_perf=True, - work_dir='/data1/sora/benchmarks/temp/output', - use_cache='/data1/sora/benchmarks/temp/output', - no_timestamp=True, - limit=1, - model='', - api_url='http://localhost:30000/v1', + model='your-model-name', + api_url='https://your-openai-compatible-endpoint/v1', + api_key='your_api_key', eval_type='openai_api', - datasets=datasets, - dataset_dir='/data1/sora/benchmarks/bash/datasets', - generation_config={ - 'temperature': 0.0, - 'stream': True, - 'max_tokens': 4096, - }, - eval_batch_size=1, - judge_model_args={ - "model_id": "deepseek-v4-flash", - "api_url": "https://api.deepseek.com/v1", - "api_key": "", - "eval_type": "openai_api", - "generation_config": { - "temperature": 0.0, - "max_tokens": 1024 * 10, - }, - } + datasets=['gsm8k', 'arc'], + limit=5 ) run_task(task_cfg) ``` -## 3. 评测结果 +
💡 Tip: run_task also supports dictionaries, YAML or JSON files as configuration. -P800 模型能力评测结果。 +**Using Python Dictionary** -## 4. UI 界面 +```python +from evalscope.run import run_task -```bash -pip install flask sse_starlette -cd ./evalscope/web -npm install -npm run build -evalscope service +task_cfg = { + 'model': 'Qwen/Qwen2.5-0.5B-Instruct', + 'datasets': ['gsm8k', 'arc'], + 'limit': 5 +} +run_task(task_cfg=task_cfg) ``` -然后访问 `http://127.0.0.1:9000/dashboard`。 +**Using YAML File** (`config.yaml`) +```yaml +model: Qwen/Qwen2.5-0.5B-Instruct +datasets: + - gsm8k + - arc +limit: 5 +``` +```python +from evalscope.run import run_task + +run_task(task_cfg="config.yaml") +``` +
+ +### Output Results +After evaluation completion, you will see a report in the terminal in the following format: +```text +┌───────────────────────┬───────────┬────────────┬───────────────┬───────┬─────────┐ +│ Model │ Dataset │ Metric │ Subset │ Num │ Score │ +├───────────────────────┼───────────┼────────────┼───────────────┼───────┼─────────┤ +│ Qwen2.5-0.5B-Instruct │ gsm8k │ Accuracy ↑ │ main │ 5 │ 40% │ +├───────────────────────┼───────────┼────────────┼───────────────┼───────┼─────────┤ +│ Qwen2.5-0.5B-Instruct │ arc │ Accuracy ↑ │ ARC-Easy │ 5 │ 80% │ +├───────────────────────┼───────────┼────────────┼───────────────┼───────┼─────────┤ +│ Qwen2.5-0.5B-Instruct │ arc │ Accuracy ↑ │ ARC-Challenge │ 5 │ 40% │ +└───────────────────────┴───────────┴────────────┴───────────────┴───────┴─────────┘ +``` + +**Launch the visualization dashboard**: +```bash +pip install 'evalscope[service]' +evalscope service +``` +Visit `http://127.0.0.1:9000` to open the visualization interface. + +## 📈 Advanced Usage + +### Custom Evaluation Parameters + +You can fine-tune model loading, inference, and dataset configuration through command line parameters. + +```shell +evalscope eval \ + --model Qwen/Qwen3-0.6B \ + --model-args '{"revision": "master", "precision": "torch.float16", "device_map": "auto"}' \ + --generation-config '{"do_sample":true,"temperature":0.6,"max_tokens":512}' \ + --dataset-args '{"gsm8k": {"few_shot_num": 0, "few_shot_random": false}}' \ + --datasets gsm8k \ + --limit 10 +``` + +- `--model-args`: Model loading parameters such as `revision`, `precision`, etc. +- `--generation-config`: Model generation parameters such as `temperature`, `max_tokens`, etc. +- `--dataset-args`: Dataset configuration parameters such as `few_shot_num`, etc. + +For details, please refer to [📖 Complete Parameter Guide](https://evalscope.readthedocs.io/en/latest/get_started/parameters.html). + +### ⚔️ Arena Mode + +Arena mode evaluates model performance through pairwise battles between models, providing win rates and rankings, perfect for horizontal comparison of multiple models. + +```text +# Example evaluation results +Model WinRate (%) CI (%) +------------ ------------- --------------- +qwen2.5-72b 69.3 (-13.3 / +12.2) +qwen2.5-7b 50 (+0.0 / +0.0) +qwen2.5-0.5b 4.7 (-2.5 / +4.4) +``` +For details, please refer to [📖 Arena Mode Usage Guide](https://evalscope.readthedocs.io/en/latest/user_guides/arena.html). + +### 🖊️ Custom Dataset Evaluation + +EvalScope allows you to easily add and evaluate your own datasets. For details, please refer to [📖 Custom Dataset Evaluation Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/custom_dataset/index.html). + +## ⚡ Inference Performance Evaluation Tool + +EvalScope provides a powerful stress testing tool for evaluating the performance of large language model services. + +- **Key Metrics**: Supports throughput (Tokens/s), first token latency (TTFT), token generation latency (TPOT), etc. +- **Result Recording**: Supports recording results to `wandb` and `swanlab`. +- **Speed Benchmarks**: Can generate speed benchmark results similar to official reports. + +For details, please refer to [📖 Performance Testing Usage Guide](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test/index.html). + +

+ +

+ +## 🧪 Other Evaluation Backends +EvalScope supports launching evaluation tasks through third-party evaluation frameworks (we call them "backends") to meet diverse evaluation needs. + +- **Native**: EvalScope's default evaluation framework with comprehensive functionality. +- **OpenCompass**: Focuses on text-only evaluation. [📖 Usage Guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/opencompass_backend.html) +- **VLMEvalKit**: Focuses on multi-modal evaluation. [📖 Usage Guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/vlmevalkit_backend.html) +- **RAGEval**: Focuses on RAG evaluation, supporting Embedding and Reranker models. [📖 Usage Guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/index.html) +- **Third-party Evaluation Tools**: Supports evaluation tasks like [ToolBench](https://evalscope.readthedocs.io/en/latest/third_party/toolbench.html). + +
🏛️ Overall Architecture + +

+ +
EvalScope Overall Architecture. +

+ +1. **Input Layer** + - **Model Sources**: API models (OpenAI API), Local models (ModelScope) + - **Datasets**: Standard evaluation benchmarks (MMLU/GSM8k etc.), Custom data (MCQ/QA) + +2. **Core Functions** + - **Multi-backend Evaluation**: Native backend, OpenCompass, MTEB, VLMEvalKit, RAGAS + - **Performance Monitoring**: Supports multiple model service APIs and data formats, tracking TTFT/TPOT and other metrics + - **Tool Extensions**: Integrates Tool-Bench, Needle-in-a-Haystack, etc. + +3. **Output Layer** + - **Structured Reports**: Supports JSON, Table, Logs + - **Visualization Platform**: Supports Web Dashboard, Wandb, SwanLab + +
+ + +## ❤️ Community & Support + +Welcome to join our community to communicate with other developers and get help. + +[Discord Group](https://discord.gg/xc66bMxc4h) | WeChat Group | DingTalk Group +:-------------------------:|:-------------------------:|:-------------------------: + | | + +## 👷‍♂️ Contributing + +We welcome any contributions from the community! If you want to add new evaluation benchmarks, models, or features, please refer to our [Contributing Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/add_benchmark.html). + +Thanks to all developers who have contributed to EvalScope! + + + + + + +
+


+
+ + +## 📚 Citation + +If you use EvalScope in your research, please cite our work: +```bibtex +@misc{evalscope_2024, + title={{EvalScope}: Evaluation Framework for Large Models}, + author={ModelScope Team}, + year={2024}, + url={https://github.com/modelscope/evalscope} +} +``` diff --git a/evalscope/README_zh.md b/evalscope/README_zh.md index 55324c2..708c087 100644 --- a/evalscope/README_zh.md +++ b/evalscope/README_zh.md @@ -13,6 +13,7 @@ PyPI version PyPI - Downloads +GitHub stars Documentation Status

@@ -39,7 +40,7 @@ evalscope eval --model your-model-name --api-url $OPENAI_API_BASE_URL --api-key - **🚀 多后端集成**: 无缝集成 OpenCompass, VLMEvalKit, RAGEval 等多种评测后端，满足不同评测需求。 - **🤖 Agent 评测模式**: 在受控的多轮 AgentLoop 中驱动 GSM8K、AIME、SWE-bench Agentic 等基准；支持可插拔的策略、工具与 Docker 沙箱，每条样本完整记录 Agent Trace 并可在仪表盘中按步骤回放。 - **⚡ 推理性能测试**: 提供强大的模型服务压力测试工具，支持 TTFT, TPOT 等多项性能指标。 -- **📊 交互式报告**: 提供 WebUI 可视化界面，支持多维度模型对比、报告概览和详情查阅。 +- **📊 交互式报告**: 提供 Web Dashboard，支持多维度模型对比、报告概览和详情查阅。 - **⚔️ 竞技场模式**: 支持多模型对战 (Pairwise Battle)，直观地对模型进行排名和评估。 - **🔧 高度可扩展**: 开发者可以轻松添加自定义数据集、模型和评测指标。 @@ -74,6 +75,10 @@ EvalScope 提供交互式 Web Dashboard，支持多维度模型对比和深入 ## 🎉 内容更新 +- 🔥 **[2026.08.24] v1.11.0** 引入可发布的评测版本标识，保障基准结果可复现；优化评测报告语义与不完整运行处理，并增强多模态媒体加载和任务配置校验。 +- 🔥 **[2026.08.13]** 评测报告升级：统一指标语义，并改善 Agent Trace 步骤分组与工具调用/结果关联的可靠性。 +- 🔥 **[2026.08.10]** 新增 **AutomationBench**、**JobBench**、**MiniWoB**、**OmniDocBench-v1.6**、**PerceptionBench**、**ScreenSpot-Pro**、**PLawBench**、**PMC-VQA**、**HiPhO**、**LogicVista** 和 **CC-OCR-V2** 基准。 +- 🔥 **[2026.07.21]** 新增 **Claw-Eval**、**ResearchRubrics**、**Toolathlon**（Agent）、**TVBench**（视频）、**WideSearch**、**PerspectiveGap** 基准。 - 🔥 **[2026.07.03]** 新增 **CharXiv** & **BabyVision**（图表理解、视觉认知）及 **ERQA** & **WorldVQA**（实体识别 QA，支持 LLM judge + CoT）多模态基准。 - 🔥 **[2026.06.23]** Agent 与代码评测大幅扩展：新增 **BigCodeBench**、**SWE-bench Multilingual**、**BrowseComp**、**MCP-Atlas**、**GDPval** 基准；新增 **OpenCode** / **OpenHands** runner；适配器架构重构，引入 `AudioLanguageAdapter`、统一 `FunctionCallAdapter`、公开 `run_agent_loop` API。 - 🔥 **[2026.06.16]** 新增全参考**图像质量指标**（SSIM、PSNR 等）、长上下文基准（**LoCoMo QA**、**LongMemEval**）、**Caption** & **Maritime-OCR-Bench** 基准；Perf 模块支持统一 `--data-source` 参数及并行化请求生成。 @@ -83,14 +88,16 @@ EvalScope 提供交互式 Web Dashboard，支持多维度模型对比和深入 - 🔥 **[2026.05.26]** 新增 [GAIA](https://evalscope.readthedocs.io/zh-cn/latest/third_party/gaia.html) agent 基准（Docker sandbox 内多轮 ReAct + `bash`，复用官方规则评分器）和通用 [MCP 服务器](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/agent/native.html#mcp-工具接入)接入 —— 任何基于 `NativeAgentConfig` 的 benchmark 都可直接挂载 stdio / HTTP / SSE 的 MCP server（`fetch`、网页搜索、GitHub 等），无需 benchmark 端改动。 - 🔥 **[2026.05.22]** 新增 **外部 Agent Bridge** 模式：可直接评测 Anthropic [Claude Code](https://github.com/anthropics/claude-code)、OpenAI [Codex](https://github.com/openai/codex) 等成品 Agent CLI。Bridge 透明转发 CLI 的 LLM 请求（Anthropic Messages / OpenAI Chat / OpenAI Responses，含 SSE 流式响应）到评测模型，同时把完整交互轨迹录制为 `agent_trace`；通过 `@register_runner` 可接入任意第三方 CLI。详见[外部 Agent Bridge 指南](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/agent/bridge.html)。 - 🔥 **[2026.05.19]** 新增对 [SWE-bench_Pro](https://evalscope.readthedocs.io/zh-cn/latest/third_party/swe_bench_pro.html) 与 [τ³-bench](https://evalscope.readthedocs.io/zh-cn/latest/third_party/tau3_bench.html) 的支持：SWE-bench_Pro 是 Scale AI 推出的更具挑战性的多语言、长周期软件工程基准，相比原始 SWE-bench 数据污染更少、覆盖语言更广，**推荐替代原始 SWE-bench 使用**，每个实例的 Docker 镜像直接从 DockerHub 拉取，无需本地构建；τ³-bench 是 tau-bench 系列的 v1.0.0 版本，在 τ²-bench 基础上新增 `banking_knowledge` 知识检索领域（RAG）、修复 75+ 项任务，并提供可插拔的检索流水线（BM25 / 稠密嵌入 / 重排序器 / 沙箱 shell）。 + +

更多历史更新 + - 🔥 **[2026.05.15]** 新增 **Agent 评测模式**：所有基于 `DefaultDataAdapter` 的基准（GSM8K、AIME、IFEval 等）现在均可通过多轮 AgentLoop 驱动，支持可插拔策略（`function_calling` / `react` / `swe_bench_*`）、工具（`bash` / `python_exec` / `submit`）以及 `local` / `docker` 运行环境，每条样本的 `agent_trace` 会随评测结果落盘，并在仪表盘的预测视图中按步骤回放。详见[Agent 评测指南](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/agent/native.html)。 - 🔥 **[2026.05.08]** 与 [LightSeek](https://lightseek.org/) 联合推出 [TokenSpeed](https://lightseek.org/blog/lightseek-tokenspeed.html)——面向 Agentic 工作负载的极速 LLM 推理引擎。EvalScope 提供 SWE-smith 压测流水线，基于真实 Coding Agent 轨迹衡量单 GPU 吞吐（TPM）与单用户延迟（TPS），作为 TokenSpeed 性能评测的官方基准工具。参考 [SWE-smith 使用指南](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/stress_test/multi_turn.html#swe-smith) 快速上手。 + - 🔥 **[2026.05.07]** 全新 Web 界面升级：使用 React + Vite 重构可视化平台，替换原有 Gradio 界面，提供更流畅的交互体验。 - 🔥 **[2026.04.23]** 支持在评测任务中记录性能（perf）指标，可在单次评测运行中同时追踪模型准确率与 TTFT、TPOT、吞吐量等推理效率指标。 - 🔥 **[2026.04.17]** 支持多轮对话性能压测，可对具备多轮上下文的对话模型服务进行负载测试，参考[使用文档](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/stress_test/examples.html)。 -
更多历史更新 - - 🔥 **[2026.04.10]** 新增支持 [TIR-Bench](https://arxiv.org/abs/2511.01833)（Thinking-with-Images Reasoning Benchmark），一个面向视觉语言模型的多模态推理基准。 - 🔥 **[2026.03.24]** 支持 Agent Skill，任何支持 Skill/Tool 调用的 Agent 模型均可通过自然语言直接驱动 EvalScope 完成模型评测、性能压测和结果可视化。 - 🔥 **[2026.03.09]** 支持评测进度追踪和自动生成HTML格式可视化报告。 @@ -247,15 +254,15 @@ run_task(task_cfg="config.yaml") ### 输出结果 评测完成后，您将在终端看到如下格式的报告： ```text -+-----------------------+----------------+-----------------+-----------------+---------------+-------+---------+ -| Model Name | Dataset Name | Metric Name | Category Name | Subset Name | Num | Score | -+=======================+================+=================+=================+===============+=======+=========+ -| Qwen2.5-0.5B-Instruct | gsm8k | AverageAccuracy | default | main | 5 | 0.4 | -+-----------------------+----------------+-----------------+-----------------+---------------+-------+---------+ -| Qwen2.5-0.5B-Instruct | ai2_arc | AverageAccuracy | default | ARC-Easy | 5 | 0.8 | -+-----------------------+----------------+-----------------+-----------------+---------------+-------+---------+ -| Qwen2.5-0.5B-Instruct | ai2_arc | AverageAccuracy | default | ARC-Challenge | 5 | 0.4 | -+-----------------------+----------------+-----------------+-----------------+---------------+-------+---------+ +┌───────────────────────┬───────────┬────────────┬───────────────┬───────┬─────────┐ +│ Model │ Dataset │ Metric │ Subset │ Num │ Score │ +├───────────────────────┼───────────┼────────────┼───────────────┼───────┼─────────┤ +│ Qwen2.5-0.5B-Instruct │ gsm8k │ Accuracy ↑ │ main │ 5 │ 40% │ +├───────────────────────┼───────────┼────────────┼───────────────┼───────┼─────────┤ +│ Qwen2.5-0.5B-Instruct │ arc │ Accuracy ↑ │ ARC-Easy │ 5 │ 80% │ +├───────────────────────┼───────────┼────────────┼───────────────┼───────┼─────────┤ +│ Qwen2.5-0.5B-Instruct │ arc │ Accuracy ↑ │ ARC-Challenge │ 5 │ 40% │ +└───────────────────────┴───────────┴────────────┴───────────────┴───────┴─────────┘ ``` **启动可视化面板**： @@ -341,7 +348,7 @@ EvalScope 支持通过第三方评测框架（我们称之为"后端"）发起 2. **核心功能** - **多后端评估**: 原生后端、OpenCompass、MTEB、VLMEvalKit、RAGAS - - **性能监控**: 支持多种模型服务 API 和数据格式，追踪 TTFT/TPOP 等指标 + - **性能监控**: 支持多种模型服务 API 和数据格式，追踪 TTFT/TPOT 等指标 - **工具扩展**: 集成 Tool-Bench, Needle-in-a-Haystack 等 3. **输出层** @@ -386,7 +393,3 @@ EvalScope 支持通过第三方评测框架（我们称之为"后端"）发起 url={https://github.com/modelscope/evalscope} } ``` - -## ⭐ Star History - -[![Star History Chart](https://api.star-history.com/svg?repos=modelscope/evalscope&type=Date)](https://star-history.com/#modelscope/evalscope&Date) diff --git a/evalscope/custom_eval/multimodal/vqa/example_placeholder.jsonl b/evalscope/custom_eval/multimodal/vqa/example_placeholder.jsonl new file mode 100644 index 0000000..af47417 --- /dev/null +++ b/evalscope/custom_eval/multimodal/vqa/example_placeholder.jsonl @@ -0,0 +1,5 @@ +{"messages": [{"role": "user", "content": "What animal is this?"}], "image_1": "custom_eval/multimodal/images/dog.jpg", "answer": "Dog"} +{"messages": [{"role": "user", "content": "What building is this?"}], "image_1": "custom_eval/multimodal/images/AMNH.jpg", "answer": "Museum"} +{"messages": [{"role": "user", "content": "Which city's skyline is this?"}], "image_1": "custom_eval/multimodal/images/tokyo.jpg", "answer": "Tokyo"} +{"messages": [{"role": "user", "content": "What is the brand of this car?"}], "image_1": "custom_eval/multimodal/images/tesla.jpg", "answer": "Tesla"} +{"messages": [{"role": "user", "content": "What is the person in the picture doing?"}], "image_1": "custom_eval/multimodal/images/running.jpg", "answer": "Running"} \ No newline at end of file diff --git a/evalscope/docs/en/advanced_guides/add_benchmark.md b/evalscope/docs/en/advanced_guides/add_benchmark.md index d591a8b..d148422 100644 --- a/evalscope/docs/en/advanced_guides/add_benchmark.md +++ b/evalscope/docs/en/advanced_guides/add_benchmark.md @@ -47,7 +47,7 @@ DataAdapter adopts a Pipeline architecture, supporting custom behavior through h calculate_metrics() ├── filter_prediction() │ └── extract_answer() [Optional User Implementation] - ├── match_score() / llm_match_score() + ├── match_score() / score_with_judge_contracts() └── Returns SampleScore 4. Result Aggregation Phase @@ -61,6 +61,31 @@ DataAdapter adopts a Pipeline architecture, supporting custom behavior through h └── Returns Report ``` +### Adding an LLM-Judged Benchmark + +Users enable judging with the typed `judge` configuration described in [Judge Parameters](../get_started/parameters.md#judge-parameters): + +```python +TaskConfig( + model='MODEL_UNDER_TEST', + datasets=['your_benchmark'], + judge={ + 'strategy': 'llm', + 'models': {'model_id': 'JUDGE_MODEL', 'api_url': 'OPENAI_COMPATIBLE_URL', 'api_key': 'API_KEY'}, + }, +) +``` + +For an adapter author, judge I/O belongs to `evalscope.api.judge`; do not call `self.llm_judge.judge()` or parse a model reply in the adapter. + +1. Declare `scoring_policy`: use `JUDGE_ONLY` when rule scoring is not meaningful, `JUDGE_DEFAULT` when rules remain available but `auto` should judge, and `RULE_DEFAULT` when `auto` should keep rule scoring. +2. Implement the single adapter entry point, `judge_definition(context)`. For an ordinary one-verdict task, return `JudgeDefinition.labels(...)` or `JudgeDefinition.numeric(...)` with a Pydantic verdict schema. These helpers append the JSON output instruction and keep the prompt, schema, and metric mapping together. +3. For a rubric, multiple claims, or staged task, define a Pydantic verdict schema and `OutputContract`, then return `JudgeDefinition.workflow(cases=..., request=..., reduce=...)`. Pass `expand=...`, `fallback=...`, or `finalize=...` only when the workflow needs them. The callbacks can be nested in `judge_definition()` or private adapter helpers, but they must be owned by the returned definition rather than exposed as adapter hooks. Append `case.output_contract.instruction()` in a custom request unless the official fixed JSON instruction exactly matches that schema. Use `CaseVerdict.metadata` rather than encoding state into `case_id`. +4. When a deterministic rule settles a sample without a model call, return `JudgeDefinition.skip(score, reason='...')`. `reason` is required and is persisted as `Score.metadata['judge_skip_reason']` with `Score.metadata['judge_skipped'] = True`; the web review panel labels this as rule-based scoring rather than an LLM verdict. +5. Add a scripted-judge test covering a valid JSON verdict, malformed/prose output, and a transport error. Invalid judge replies are excluded from the metric; they are not converted to a zero score and are not automatically retried by the executor. + +The executor owns request dispatch, position swaps, repeats, multi-judge quorum, aggregation, and review diagnostics. The model transport owns its own retry policy through `generation_config`. + ### Core Data Structures #### 1. Sample Object @@ -108,38 +133,38 @@ class ModelOutput: Represents the scoring result of a single sample: ```python -@dataclass -class Score: - value: Dict[str, float] # Scores for each metric {"acc": 1.0, "f1": 0.8} - extracted_prediction: str # Extracted prediction answer - prediction: str # Raw prediction text - metadata: Dict = None # Scoring metadata +class Score(BaseModel): + value: Dict[str, int | float | bool] = Field(default_factory=dict) # E.g. {"accuracy": 1.0} + extracted_prediction: Optional[str] = None + prediction: Optional[str] = None + explanation: Optional[str] = None + metadata: Optional[Dict] = Field(default_factory=dict) + main_score_name: Optional[str] = None # Selects one value within this sample only ``` #### 5. SampleScore Object Encapsulates the complete scoring information of a single sample: ```python -@dataclass -class SampleScore: - score: Score # Scoring object - sample_id: Optional[str] # Unique identifier for the sample - group_id: Optional[str] # Group identifier - sample_metadata: Optional[Dict] = None # Sample metadata +class SampleScore(BaseModel): + score: Score + sample_id: Optional[str | int] = None + group_id: Optional[str | int] = None + sample_metadata: Optional[Dict] = None ``` #### 6. AggScore Object Represents aggregated scoring statistics: ```python -@dataclass -class AggScore: - metric: str # Metric name - value: float # Aggregated value (e.g., average score) - subset: str # Subset name - num_samples: int # Number of samples - agg_method: str # Aggregation method (mean, median, etc.) - metadata: Dict = None # Aggregation metadata +class AggScore(BaseModel): + score: float = 0.0 + metric_name: str = '' # Canonical measured concept, e.g. "accuracy" + aggregation: str = 'identity' + dimensions: Dict[str, str | int | float | bool] = Field(default_factory=dict) + num: int = 0 + ids: Optional[List[str | int]] = None + metadata: Optional[Dict] = None ``` #### 7. DatasetDict Object @@ -325,7 +350,7 @@ Reasoning: few_shot_num=4, # Few-shot example number train_split='train', # Training set split name eval_split='test', # Evaluation set split name - metric_list=['acc'], # Evaluation metrics + metric_list=['accuracy'], # Canonical evaluation metrics prompt_template=PROMPT_TEMPLATE, # Prompt template ) ) @@ -400,7 +425,7 @@ SUBSET_LIST = [ description='MMLU-Pro is a benchmark for evaluating language models on multiple-choice questions across various subjects.', dataset_id='modelscope/MMLU-Pro', subset_list=SUBSET_LIST, - metric_list=['acc'], + metric_list=['accuracy'], few_shot_num=5, train_split='validation', eval_split='test', @@ -468,6 +493,67 @@ class MMLUProAdapter(MultiChoiceAdapter): - General Text Reasoning: Focuses more on guiding the reasoning process - Multiple Choice: Focuses on displaying choices and answer format +### Metric Semantics and the Primary Metric + +Reports do not guess what a metric means. How a metric is displayed — its name, its optimization +direction, its unit, its scale and its precision — comes from a central catalog at +`evalscope/metrics/semantics/catalog.py`, and each benchmark states which of its metrics carries +the conclusion. + +**Most new benchmarks need no catalog change at all.** Reusing an existing canonical metric name +(`accuracy`, `f1`, `exact_match`, `pass_rate`, ...) means the semantics are already declared: + +```python +metric_list=['accuracy'], +``` + +Two cases are worth a line from you: + +1. **Your benchmark reports several metrics or several variants of one metric.** Declare exactly + which emitted identity is primary. A selector may constrain the aggregation and any structured + dimensions such as `k`, `scope`, or `threshold`: + + ```python + from evalscope.api.metric.semantics import MetricSelector + + metric_list=['precision', 'recall', 'f1', 'accuracy'], + primary_metric=MetricSelector(name='f1', aggregation='mean'), + ``` + + A single non-diagnostic identity is implicitly primary. If several non-diagnostic identities + are emitted, omitting the selector makes report generation fail instead of guessing from list + order. A selector must match exactly one emitted identity, and its name must be declared in + `metric_list`. + +2. **Your benchmark introduces a new canonical metric name.** Add one line to `METRIC_DEFINITIONS`, + referencing the baseline that describes it: + + ```python + # evalscope/metrics/semantics/catalog.py + METRIC_DEFINITIONS['my_new_score'] = MetricEntry(baseline='quality.accuracy.ratio') + ``` + +Keep the naming layers separate: + +- `metric_list`, `Score.value`, and custom `AggScore.metric_name` use canonical names such as + `accuracy`. A small set of legacy aliases is normalized for compatibility, but new adapters + should not introduce more aliases. +- `AggScore` stores `metric_name`, `aggregation`, and `dimensions` separately. Do not encode + `mean`, `pass@k`, thresholds, or scopes into the metric name. +- The catalog is keyed by the canonical metric name. Aggregation-specific meaning belongs in + `AGGREGATION_SEMANTICS`; benchmark-specific name collisions belong in + `BENCHMARK_METRIC_OVERRIDES`. +- `Score.main_score_name` selects one value in a sample, `BenchmarkMeta.primary_metric` declares the + report-level primary identity, and `Report.primary_metric_identity` persists that identity. + +After changing `primary_metric`, refresh its generated metadata cache with +`make docs-update BENCHMARK="" FORCE=1`; do not edit `_meta/*.json` by hand. + +An undeclared metric degrades to a diagnostic, which displays the stored value without claiming a +direction or unit and logs the catalog entry to add. Dynamic variants do not require catalog +enumeration: values such as `k`, question type, threshold, and token range belong in structured +dimensions and share the canonical metric's semantics. + ## 4. Running Evaluation Debug the code to see if it can run normally. @@ -505,11 +591,11 @@ Output Example: +-----------------------+-----------+-----------------+------------------+-------+---------+---------+ | Model | Dataset | Metric | Subset | Num | Score | Cat.0 | +=======================+===========+=================+==================+=======+=========+=========+ -| Qwen2.5-0.5B-Instruct | gsm8k | mean_acc | main | 10 | 0.3 | default | +| Qwen2.5-0.5B-Instruct | gsm8k | Accuracy ↑ | main | 10 | 30% | default | +-----------------------+-----------+-----------------+------------------+-------+---------+---------+ -| Qwen2.5-0.5B-Instruct | mmlu_pro | mean_acc | computer science | 10 | 0.1 | default | +| Qwen2.5-0.5B-Instruct | mmlu_pro | Accuracy ↑ | computer science | 10 | 10% | default | +-----------------------+-----------+-----------------+------------------+-------+---------+---------+ -| Qwen2.5-0.5B-Instruct | mmlu_pro | mean_acc | math | 10 | 0.1 | default | +| Qwen2.5-0.5B-Instruct | mmlu_pro | Accuracy ↑ | math | 10 | 10% | default | +-----------------------+-----------+-----------------+------------------+-------+---------+---------+ ``` @@ -525,8 +611,8 @@ make docs ``` ## 6. Submitting PR -After completing the implementation of these methods and document generation, your benchmark evaluation is ready! You can submit a [PR](https://github.com/modelscope/evalscope/pulls). Before submitting, please run the following command, which will automatically format the code: +After completing the implementation and documentation generation, run all repository checks before submitting a [PR](https://github.com/modelscope/evalscope/pulls). This command applies safe Ruff fixes and formatting before validating the remaining hooks: ```bash make lint ``` -Ensure there are no formatting issues, and we will merge your contribution as soon as possible, allowing more users to use the benchmark evaluation you contributed. If you don't know how to submit a PR, you can check our [Guide](https://github.com/modelscope/evalscope/blob/main/CONTRIBUTING.md). Give it a try 🚀 \ No newline at end of file +Once the checks pass, your contribution is ready for review. For the complete development workflow, see the [Contributing Guide](https://github.com/modelscope/evalscope/blob/main/CONTRIBUTING.md). Give it a try 🚀 diff --git a/evalscope/docs/en/advanced_guides/custom_dataset/clip.md b/evalscope/docs/en/advanced_guides/custom_dataset/clip.md index 1fdfe8a..b1209d3 100644 --- a/evalscope/docs/en/advanced_guides/custom_dataset/clip.md +++ b/evalscope/docs/en/advanced_guides/custom_dataset/clip.md @@ -46,7 +46,7 @@ task_cfg = { } ``` ```{seealso} -[Full Parameter Explanation](../../user_guides/backend/rageval_backend/clip_benchmark.md#configure-evaluation-parameters) +[Full Parameter Reference](../../user_guides/backend/rageval_backend/clip_benchmark.md#full-parameter-reference) ``` Where: diff --git a/evalscope/docs/en/advanced_guides/custom_dataset/embedding.md b/evalscope/docs/en/advanced_guides/custom_dataset/embedding.md index cb1a784..7d0b093 100644 --- a/evalscope/docs/en/advanced_guides/custom_dataset/embedding.md +++ b/evalscope/docs/en/advanced_guides/custom_dataset/embedding.md @@ -148,7 +148,7 @@ Each task in the `custom_tasks` list corresponds to a `CustomTaskConfig` with th | `eval_splits` | List[str] | `["test"]` | Evaluation splits list | ```{note} -Other evaluation parameters (such as `models`, `limits`, `overwrite_results`, etc.) are consistent with the default configuration. See [MTEB Evaluation Parameter Reference](../../user_guides/backend/rageval_backend/mteb.md#parameter-explanation) for details. +Other evaluation parameters (such as `models`, `limits`, `overwrite_results`, etc.) are consistent with the default configuration. See [MTEB Evaluation Parameter Reference](../../user_guides/backend/rageval_backend/mteb.md#full-parameter-reference) for details. ``` ## FAQ diff --git a/evalscope/docs/en/advanced_guides/custom_dataset/llm.md b/evalscope/docs/en/advanced_guides/custom_dataset/llm.md index 7c273f2..5f887aa 100644 --- a/evalscope/docs/en/advanced_guides/custom_dataset/llm.md +++ b/evalscope/docs/en/advanced_guides/custom_dataset/llm.md @@ -82,7 +82,7 @@ Results: +---------------------+-------------+-----------------+----------+-------+---------+---------+ | Model | Dataset | Metric | Subset | Num | Score | Cat.0 | +=====================+=============+=================+==========+=======+=========+=========+ -| Qwen2-0.5B-Instruct | general_mcq | AverageAccuracy | example | 12 | 0.5833 | default | +| Qwen2-0.5B-Instruct | general_mcq | Accuracy ↑ | example | 12 | 58.3% | default | +---------------------+-------------+-----------------+----------+-------+---------+---------+ ``` @@ -209,45 +209,51 @@ run_task(task_cfg=task_cfg) +----------------+------------+-----------+----------+-------+---------+---------+ | Model | Dataset | Metric | Subset | Num | Score | Cat.0 | +================+============+===========+==========+=======+=========+=========+ -| Qwen2.5-0.5B-Instruct | general_qa | Rouge-1-R | example | 12 | 0.694 | default | +| Qwen2.5-0.5B-Instruct | General-QA | ROUGE ↑ · 1 · Recall | example | 12 | 69.4% | default | +----------------+------------+-----------+----------+-------+---------+---------+ -| Qwen2.5-0.5B-Instruct | general_qa | Rouge-1-P | example | 12 | 0.176 | default | +| Qwen2.5-0.5B-Instruct | General-QA | ROUGE ↑ · 1 · Precision | example | 12 | 17.6% | default | +----------------+------------+-----------+----------+-------+---------+---------+ -| Qwen2.5-0.5B-Instruct | general_qa | Rouge-1-F | example | 12 | 0.2276 | default | +| Qwen2.5-0.5B-Instruct | General-QA | ROUGE ↑ · 1 · F1 | example | 12 | 22.8% | default | +----------------+------------+-----------+----------+-------+---------+---------+ -| Qwen2.5-0.5B-Instruct | general_qa | Rouge-2-R | example | 12 | 0.4667 | default | +| Qwen2.5-0.5B-Instruct | General-QA | ROUGE ↑ · 2 · Recall | example | 12 | 46.7% | default | +----------------+------------+-----------+----------+-------+---------+---------+ -| Qwen2.5-0.5B-Instruct | general_qa | Rouge-2-P | example | 12 | 0.0939 | default | +| Qwen2.5-0.5B-Instruct | General-QA | ROUGE ↑ · 2 · Precision | example | 12 | 9.4% | default | +----------------+------------+-----------+----------+-------+---------+---------+ -| Qwen2.5-0.5B-Instruct | general_qa | Rouge-2-F | example | 12 | 0.1226 | default | +| Qwen2.5-0.5B-Instruct | General-QA | ROUGE ↑ · 2 · F1 | example | 12 | 12.3% | default | +----------------+------------+-----------+----------+-------+---------+---------+ -| Qwen2.5-0.5B-Instruct | general_qa | Rouge-L-R | example | 12 | 0.6528 | default | +| Qwen2.5-0.5B-Instruct | General-QA | ROUGE ↑ · L · Recall | example | 12 | 65.3% | default | +----------------+------------+-----------+----------+-------+---------+---------+ -| Qwen2.5-0.5B-Instruct | general_qa | Rouge-L-P | example | 12 | 0.1628 | default | +| Qwen2.5-0.5B-Instruct | General-QA | ROUGE ↑ · L · Precision | example | 12 | 16.3% | default | +----------------+------------+-----------+----------+-------+---------+---------+ -| Qwen2.5-0.5B-Instruct | general_qa | Rouge-L-F | example | 12 | 0.2063 | default | +| Qwen2.5-0.5B-Instruct | General-QA | ROUGE ↑ · L · F1 | example | 12 | 20.6% | default | +----------------+------------+-----------+----------+-------+---------+---------+ -| Qwen2.5-0.5B-Instruct | general_qa | bleu-1 | example | 12 | 0.164 | default | +| Qwen2.5-0.5B-Instruct | General-QA | BLEU ↑ · 1 | example | 12 | 16.4% | default | +----------------+------------+-----------+----------+-------+---------+---------+ -| Qwen2.5-0.5B-Instruct | general_qa | bleu-2 | example | 12 | 0.0935 | default | +| Qwen2.5-0.5B-Instruct | General-QA | BLEU ↑ · 2 | example | 12 | 9.4% | default | +----------------+------------+-----------+----------+-------+---------+---------+ -| Qwen2.5-0.5B-Instruct | general_qa | bleu-3 | example | 12 | 0.065 | default | +| Qwen2.5-0.5B-Instruct | General-QA | BLEU ↑ · 3 | example | 12 | 6.6% | default | +----------------+------------+-----------+----------+-------+---------+---------+ -| Qwen2.5-0.5B-Instruct | general_qa | bleu-4 | example | 12 | 0.0556 | default | +| Qwen2.5-0.5B-Instruct | General-QA | BLEU ↑ · 4 | example | 12 | 5.6% | default | +----------------+------------+-----------+----------+-------+---------+---------+ ```
**Method 2: Evaluation based on LLM** -LLM-based evaluation can conveniently assess the correctness of model outputs (or other dimensions of metrics, requiring custom prompt settings). Below is an example configuring `judge_model_args` parameters, using the preset `pattern` mode to determine the correctness of model outputs. +LLM-based evaluation can conveniently assess the correctness of model outputs (or other dimensions of metrics, requiring custom prompt settings). Below is an example using the preset `pattern` JSON contract to determine correctness. For a complete explanation of judge parameters, please refer to [documentation](../../get_started/parameters.md#judge-parameters). +```{note} +The judge replies with a single JSON object. A reply that cannot be read as one is **excluded** from +the metric rather than scored 0, so `Num` may be lower than the sample count. A +custom `prompt_template` should state the grading criteria only; the reply format is appended +automatically. +``` + ```python import os from evalscope import TaskConfig, run_task -from evalscope.constants import JudgeStrategy task_cfg = TaskConfig( model='Qwen/Qwen2.5-0.5B-Instruct', @@ -262,22 +268,18 @@ task_cfg = TaskConfig( ], } }, - # judge related parameters - judge_model_args={ - 'model_id': 'qwen2.5-72b-instruct', - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': os.getenv('DASHSCOPE_API_KEY'), - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 4096 + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'qwen2.5-72b-instruct', + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': os.getenv('DASHSCOPE_API_KEY'), + 'generation_config': {'temperature': 0.0, 'max_tokens': 4096}, }, - # Determine if the model output is correct based on reference answers and model output - 'score_type': 'pattern', + 'contract': {'score_type': 'pattern'}, }, # eval concurrency number eval_batch_size=5, - # Use LLM for evaluation - judge_strategy=JudgeStrategy.LLM, ) run_task(task_cfg=task_cfg) @@ -289,7 +291,7 @@ run_task(task_cfg=task_cfg) +----------------+------------+----------------+----------+-------+---------+---------+ | Model | Dataset | Metric | Subset | Num | Score | Cat.0 | +================+============+================+==========+=======+=========+=========+ -| Qwen2.5-0.5B-Instruct | general_qa | AverageAccuracy | example | 12 | 0.583 | default | +| Qwen2.5-0.5B-Instruct | General-QA | Accuracy ↑ | example | 12 | 58.3% | default | +----------------+------------+----------------+----------+-------+---------+---------+ ```
@@ -298,13 +300,12 @@ run_task(task_cfg=task_cfg) If the dataset lacks reference answers, an LLM judge can be used to evaluate the model's output answers. Without configuring an LLM, no scoring results will be available. -Below is an example configuring `judge_model_args` parameters, using the preset `numeric` mode to automatically assess model output scores from dimensions such as accuracy, relevance, and usefulness. Higher scores indicate better model output. +Below is an example using the preset `numeric` JSON contract to automatically assess model output scores from dimensions such as accuracy, relevance, and usefulness. Higher scores indicate better model output. For a complete explanation of judge parameters, please refer to [documentation](../../get_started/parameters.md#judge-parameters). ```python import os from evalscope import TaskConfig, run_task -from evalscope.constants import JudgeStrategy task_cfg = TaskConfig( model='Qwen/Qwen2.5-0.5B-Instruct', @@ -319,22 +320,18 @@ task_cfg = TaskConfig( ], } }, - # judge related parameters - judge_model_args={ - 'model_id': 'qwen2.5-72b-instruct', - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': os.getenv('DASHSCOPE_API_KEY'), - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 4096 + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'qwen2.5-72b-instruct', + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': os.getenv('DASHSCOPE_API_KEY'), + 'generation_config': {'temperature': 0.0, 'max_tokens': 4096}, }, - # Direct scoring - 'score_type': 'numeric', + 'contract': {'score_type': 'numeric'}, }, # eval concurrency number eval_batch_size=5, - # Use LLM for evaluation - judge_strategy=JudgeStrategy.LLM, ) run_task(task_cfg=task_cfg) @@ -346,7 +343,7 @@ run_task(task_cfg=task_cfg) +----------------+------------+----------------+----------+-------+---------+---------+ | Model | Dataset | Metric | Subset | Num | Score | Cat.0 | +================+============+================+==========+=======+=========+=========+ -| Qwen2.5-0.5B-Instruct | general_qa | AverageAccuracy | example | 12 | 0.6375 | default | +| Qwen2.5-0.5B-Instruct | General-QA | Accuracy ↑ | example | 12 | 63.8% | default | +----------------+------------+----------------+----------+-------+---------+---------+ ``` @@ -495,4 +492,4 @@ Example output: +-----------+------------+-------------------------------+----------+-------+---------+---------+ | qwen-plus | general_fc | tool_call_f1 | default | 10 | 0.5 | default | +-----------+------------+-------------------------------+----------+-------+---------+---------+ -``` \ No newline at end of file +``` diff --git a/evalscope/docs/en/advanced_guides/custom_dataset/vlm.md b/evalscope/docs/en/advanced_guides/custom_dataset/vlm.md index 663fda4..91bdadb 100644 --- a/evalscope/docs/en/advanced_guides/custom_dataset/vlm.md +++ b/evalscope/docs/en/advanced_guides/custom_dataset/vlm.md @@ -2,22 +2,29 @@ This framework supports two custom multimodal evaluation methods: -- **General-VQA Format**: Based on OpenAI message format, supports multi-image/audio input, system prompts, and base64 encoding, suitable for Q&A-based multimodal evaluation tasks. -- **General-VMCQ Format**: Similar to MMMU format, question text can contain image placeholders ``, suitable for multiple-choice multimodal evaluation tasks. +- **General-VQA Format**: Suitable for Q&A-based multimodal evaluation tasks. Supports two input styles: **OpenAI Messages Data**, **MMMU-style Data with Media Placeholders**. +- **General-VMCQ Format**: Suitable for multiple-choice multimodal evaluation tasks. Uses the [media placeholders][mp-feature] to embed images, videos, and audio in questions and options, similar to MMMU format. ## General-VQA Format -### 1. Data Preparation +General-VQA supports **two input styles**: -Prepare data files conforming to OpenAI message format, supporting **JSONL** or **TSV** formats: +1. **OpenAI Messages Data** — full structured content with explicit media parts (images, audio, video) in the OpenAI message schema. Supports multi-turn conversations, system prompts, and fine-grained control over each content part. +2. **MMMU-style Data with Media Placeholders** — a simpler approach where the user message is a plain-text string containing ``, `