sora/evalstone
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
evalstone/evalscope/tests/report/semantics/golden_samples.json
sora 13274243a0 Bump vendored EvalScope and add K3-ready DPV4 configs. 
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-02 07:30:48 +00:00

608 lines
16 KiB
JSON
Raw Permalink Blame History

{
"semantics": {
"accuracy": {
"semantic_id": "quality.accuracy.ratio",
"metric_name": "Accuracy",
"display_name": null,
"kind": "quality",
"direction": "higher_is_better",
"raw_unit": null,
"value_range": {
"min": 0.0,
"max": 1.0
},
"display_kind": "percent",
"display_multiplier": 100.0,
"display_unit": "%",
"display_precision": 1
},
"accuracy_precision_0": {
"semantic_id": "quality.accuracy.ratio",
"metric_name": "Accuracy",
"display_name": null,
"kind": "quality",
"direction": "higher_is_better",
"raw_unit": null,
"value_range": {
"min": 0.0,
"max": 1.0
},
"display_kind": "percent",
"display_multiplier": 100.0,
"display_unit": "%",
"display_precision": 0
},
"f1": {
"semantic_id": "quality.f1.ratio",
"metric_name": "F1",
"display_name": null,
"kind": "quality",
"direction": "higher_is_better",
"raw_unit": null,
"value_range": {
"min": 0.0,
"max": 1.0
},
"display_kind": "percent",
"display_multiplier": 100.0,
"display_unit": "%",
"display_precision": 1
},
"points_100": {
"semantic_id": "quality.score.points_100",
"metric_name": "Score",
"display_name": null,
"kind": "quality",
"direction": "higher_is_better",
"raw_unit": null,
"value_range": {
"min": 0.0,
"max": 100.0
},
"display_kind": "percent",
"display_multiplier": 1.0,
"display_unit": "%",
"display_precision": 1
},
"wer": {
"semantic_id": "quality.wer.ratio",
"metric_name": "WER",
"display_name": null,
"kind": "quality",
"direction": "lower_is_better",
"raw_unit": null,
"value_range": {
"min": 0.0,
"max": 1.0
},
"display_kind": "percent",
"display_multiplier": 100.0,
"display_unit": "%",
"display_precision": 1
},
"parse_status": {
"semantic_id": "diagnostic.parse_status.ratio",
"metric_name": "Parse Status",
"display_name": null,
"kind": "diagnostic",
"direction": "none",
"raw_unit": null,
"value_range": {
"min": 0.0,
"max": 1.0
},
"display_kind": "percent",
"display_multiplier": 100.0,
"display_unit": "%",
"display_precision": 1
},
"incorrect_rate": {
"semantic_id": "diagnostic.parse_status.ratio",
"metric_name": "Parse Status",
"display_name": "Incorrect rate",
"kind": "diagnostic",
"direction": "none",
"raw_unit": null,
"value_range": {
"min": 0.0,
"max": 1.0
},
"display_kind": "percent",
"display_multiplier": 100.0,
"display_unit": "%",
"display_precision": 1
},
"seconds": {
"semantic_id": "perf.latency.seconds",
"metric_name": "Latency",
"display_name": null,
"kind": "quality",
"direction": "lower_is_better",
"raw_unit": "s",
"value_range": null,
"display_kind": "number",
"display_multiplier": null,
"display_unit": "s",
"display_precision": 3
},
"milliseconds": {
"semantic_id": "perf.latency.milliseconds",
"metric_name": "Latency",
"display_name": null,
"kind": "quality",
"direction": "lower_is_better",
"raw_unit": "ms",
"value_range": null,
"display_kind": "number",
"display_multiplier": null,
"display_unit": "ms",
"display_precision": 2
},
"ttft_ms": {
"semantic_id": "perf.latency.seconds",
"metric_name": "TTFT",
"display_name": null,
"kind": "quality",
"direction": "lower_is_better",
"raw_unit": "s",
"value_range": null,
"display_kind": "number",
"display_multiplier": 1000.0,
"display_unit": "ms",
"display_precision": 1
},
"token_throughput": {
"semantic_id": "perf.throughput.tokens_per_second",
"metric_name": "Token Throughput",
"display_name": null,
"kind": "quality",
"direction": "higher_is_better",
"raw_unit": "tok/s",
"value_range": null,
"display_kind": "number",
"display_multiplier": null,
"display_unit": "tok/s",
"display_precision": 2
},
"judge_score": {
"semantic_id": "quality.judge_score.unbounded",
"metric_name": "Judge Score",
"display_name": null,
"kind": "quality",
"direction": "higher_is_better",
"raw_unit": null,
"value_range": null,
"display_kind": "number",
"display_multiplier": null,
"display_unit": null,
"display_precision": 2
},
"count": {
"semantic_id": "diagnostic.count.items",
"metric_name": "Count",
"display_name": null,
"kind": "diagnostic",
"direction": "none",
"raw_unit": null,
"value_range": null,
"display_kind": "number",
"display_multiplier": null,
"display_unit": null,
"display_precision": 0
},
"diagnostic_precision_2": {
"semantic_id": "diagnostic.unspecified",
"metric_name": "Value",
"display_name": null,
"kind": "diagnostic",
"direction": "none",
"raw_unit": null,
"value_range": null,
"display_kind": "number",
"display_multiplier": null,
"display_unit": null,
"display_precision": 2
},
"diagnostic_precision_0": {
"semantic_id": "diagnostic.unspecified",
"metric_name": "Value",
"display_name": null,
"kind": "diagnostic",
"direction": "none",
"raw_unit": null,
"value_range": null,
"display_kind": "number",
"display_multiplier": null,
"display_unit": null,
"display_precision": 0
},
"percent_pp": {
"semantic_id": "quality.score.ratio",
"metric_name": "Change",
"display_name": null,
"kind": "quality",
"direction": "higher_is_better",
"raw_unit": null,
"value_range": {
"min": 0.0,
"max": 1.0
},
"display_kind": "percent",
"display_multiplier": 100.0,
"display_unit": "pp",
"display_precision": 1
},
"number_percent": {
"semantic_id": "diagnostic.unspecified",
"metric_name": "Cache Hit Rate",
"display_name": null,
"kind": "diagnostic",
"direction": "none",
"raw_unit": "%",
"value_range": null,
"display_kind": "number",
"display_multiplier": null,
"display_unit": "%",
"display_precision": 2
},
"rouge": {
"semantic_id": "quality.rouge.ratio",
"metric_name": "ROUGE",
"display_name": null,
"kind": "quality",
"direction": "higher_is_better",
"raw_unit": null,
"value_range": {
"min": 0.0,
"max": 1.0
},
"display_kind": "percent",
"display_multiplier": 100.0,
"display_unit": "%",
"display_precision": 1
},
"similarity": {
"semantic_id": "quality.similarity.ratio",
"metric_name": "Similarity",
"display_name": null,
"kind": "quality",
"direction": "higher_is_better",
"raw_unit": null,
"value_range": {
"min": 0.0,
"max": 1.0
},
"display_kind": "percent",
"display_multiplier": 100.0,
"display_unit": "%",
"display_precision": 1
},
"legacy_diagnostic": {
"semantic_id": "diagnostic.unspecified",
"metric_name": "Weird.Metric",
"display_name": null,
"kind": "diagnostic",
"direction": "none",
"raw_unit": null,
"value_range": null,
"display_kind": "number",
"display_multiplier": null,
"display_unit": null,
"display_precision": 4
},
"total_tokens": {
"semantic_id": "diagnostic.unspecified",
"metric_name": "total_tokens",
"display_name": null,
"kind": "diagnostic",
"direction": "none",
"raw_unit": null,
"value_range": null,
"display_kind": "number",
"display_multiplier": null,
"display_unit": null,
"display_precision": 4
}
},
"samples": [
{
"id": "percent-ratio-basic",
"semantics_ref": "accuracy",
"value": 0.8567,
"expected_primary": "85.7%",
"expected_raw": "0.8567"
},
{
"id": "percent-ratio-positive-tie",
"semantics_ref": "accuracy_precision_0",
"value": 0.125,
"expected_primary": "13%",
"expected_raw": "0.125"
},
{
"id": "percent-ratio-zero",
"semantics_ref": "accuracy",
"value": 0.0,
"expected_primary": "0%",
"expected_raw": "0"
},
{
"id": "percent-ratio-upper-bound",
"semantics_ref": "accuracy",
"value": 1.0,
"expected_primary": "100%",
"expected_raw": "1"
},
{
"id": "percent-ratio-trailing-zero-trimmed",
"semantics_ref": "f1",
"value": 0.9,
"expected_primary": "90%",
"expected_raw": "0.9"
},
{
"id": "percent-official-points-100",
"semantics_ref": "points_100",
"value": 87.25,
"expected_primary": "87.3%",
"expected_raw": "87.25"
},
{
"id": "percent-lower-is-better-wer",
"semantics_ref": "wer",
"value": 0.0432,
"expected_primary": "4.3%",
"expected_raw": "0.0432"
},
{
"id": "percent-diagnostic-parse-status",
"semantics_ref": "parse_status",
"value": 0.9876,
"expected_primary": "98.8%",
"expected_raw": "0.9876"
},
{
"id": "number-seconds",
"semantics_ref": "seconds",
"value": 1.23456,
"expected_primary": "1.235 s",
"expected_raw": "1.2346 s"
},
{
"id": "number-seconds-rounds-to-zero",
"semantics_ref": "seconds",
"value": 0.0001234,
"expected_primary": "0 s",
"expected_raw": "0.0001 s"
},
{
"id": "number-milliseconds",
"semantics_ref": "milliseconds",
"value": 250.0,
"expected_primary": "250 ms",
"expected_raw": "250 ms"
},
{
"id": "number-seconds-displayed-as-milliseconds",
"semantics_ref": "ttft_ms",
"value": 0.700828,
"expected_primary": "700.8 ms",
"expected_raw": "0.7008 s"
},
{
"id": "number-throughput-tokens-per-second",
"semantics_ref": "token_throughput",
"value": 1234.5678,
"expected_primary": "1234.57 tok/s",
"expected_raw": "1234.5678 tok/s"
},
{
"id": "number-without-unit",
"semantics_ref": "judge_score",
"value": 7.5,
"expected_primary": "7.5",
"expected_raw": "7.5"
},
{
"id": "number-negative-without-unit",
"semantics_ref": "judge_score",
"value": -3.456,
"expected_primary": "-3.46",
"expected_raw": "-3.456"
},
{
"id": "number-diagnostic-count",
"semantics_ref": "count",
"value": 12.0,
"expected_primary": "12",
"expected_raw": "12"
},
{
"id": "missing-value-with-semantics",
"semantics_ref": "accuracy",
"value": null,
"expected_primary": "—",
"expected_raw": "—"
},
{
"id": "diagnostic-fallback-ratio",
"semantics_ref": null,
"value": 0.87654321,
"expected_primary": "0.8765",
"expected_raw": "0.8765"
},
{
"id": "diagnostic-fallback-integer",
"semantics_ref": null,
"value": 42.0,
"expected_primary": "42",
"expected_raw": "42"
},
{
"id": "number-decimal-tie",
"semantics_ref": "diagnostic_precision_2",
"value": 1.005,
"expected_primary": "1.01",
"expected_raw": "1.005"
},
{
"id": "number-negative-half-tie",
"semantics_ref": "diagnostic_precision_0",
"value": -0.5,
"expected_primary": "0",
"expected_raw": "-0.5"
},
{
"id": "diagnostic-fallback-missing-value",
"semantics_ref": null,
"value": null,
"expected_primary": "—",
"expected_raw": "—"
},
{
"id": "percent_kind_with_non_percent_unit",
"semantics_ref": "percent_pp",
"value": 0.5,
"expected_primary": "50pp",
"expected_raw": "0.5"
},
{
"id": "number_kind_with_percent_unit",
"semantics_ref": "number_percent",
"value": 12.345,
"expected_primary": "12.35 %",
"expected_raw": "12.345 %"
},
{
"id": "label-no-dimensions",
"semantics_ref": "accuracy",
"identity": {
"name": "accuracy",
"aggregation": "mean",
"dimensions": {}
},
"value": 0.8567,
"expected_primary": "85.7%",
"expected_raw": "0.8567",
"expected_label": "Accuracy ↑"
},
{
"id": "label-lower-is-better",
"semantics_ref": "wer",
"identity": {
"name": "wer",
"aggregation": "mean",
"dimensions": {}
},
"value": 0.043,
"expected_primary": "4.3%",
"expected_raw": "0.043",
"expected_label": "WER ↓"
},
{
"id": "label-single-dimension",
"semantics_ref": "accuracy",
"identity": {
"name": "accuracy",
"aggregation": "mean",
"dimensions": {
"threshold": 0.5
}
},
"value": 0.5,
"expected_primary": "50%",
"expected_raw": "0.5",
"expected_label": "Accuracy ↑ · 0.5"
},
{
"id": "label-dimension-order",
"semantics_ref": "rouge",
"identity": {
"name": "rouge",
"aggregation": "mean",
"dimensions": {
"statistic": "recall",
"variant": "l"
}
},
"value": 0.612,
"expected_primary": "61.2%",
"expected_raw": "0.612",
"expected_label": "ROUGE ↑ · L · Recall"
},
{
"id": "label-boolean-dimension",
"semantics_ref": "accuracy",
"identity": {
"name": "accuracy",
"aggregation": "mean",
"dimensions": {
"scope": "overall",
"strict": true
}
},
"value": 0.5,
"expected_primary": "50%",
"expected_raw": "0.5",
"expected_label": "Accuracy ↑ · Overall · Yes"
},
{
"id": "label-uppercase-dimension",
"semantics_ref": "similarity",
"identity": {
"name": "table_teds",
"aggregation": "mean",
"dimensions": {
"scope": "table_TEDS"
}
},
"value": 0.77,
"expected_primary": "77%",
"expected_raw": "0.77",
"expected_label": "Similarity ↑ · Table TEDS"
},
{
"id": "label-diagnostic-keeps-legacy-name",
"semantics_ref": "legacy_diagnostic",
"identity": {
"name": "legacy_metric",
"aggregation": "identity",
"dimensions": {
"original_name": "Weird.Metric"
}
},
"legacy_name": "Weird.Metric",
"value": 12.5,
"expected_primary": "12.5",
"expected_raw": "12.5",
"expected_label": "Weird.Metric"
},
{
"id": "label-diagnostic-display-name",
"semantics_ref": "incorrect_rate",
"identity": {
"name": "is_incorrect",
"aggregation": "mean",
"dimensions": {}
},
"value": 0.25,
"expected_primary": "25%",
"expected_raw": "0.25",
"expected_label": "Incorrect rate"
},
{
"id": "label-diagnostic-without-legacy-name",
"semantics_ref": "total_tokens",
"identity": {
"name": "total_tokens",
"aggregation": "mean",
"dimensions": {}
},
"value": 1234.0,
"expected_primary": "1234",
"expected_raw": "1234",
"expected_label": "total_tokens:mean"
}
]
}
Reference in New Issue View Git Blame Copy Permalink