docs: move SLO_STANDARDS.md from scripts/ to docs/; update references
This commit is contained in:
parent
ccf65e745b
commit
80959cf765
@ -13,7 +13,7 @@
|
|||||||
| `experiments/` | 以实验为单位的自包含目录(脚本 + 配置 + 结果) |
|
| `experiments/` | 以实验为单位的自包含目录(脚本 + 配置 + 结果) |
|
||||||
| `experiments/legacy_bench_results/` | 历史归档的实验产物(只读),从原 `bench_results/` 迁移而来 |
|
| `experiments/legacy_bench_results/` | 历史归档的实验产物(只读),从原 `bench_results/` 迁移而来 |
|
||||||
| `BENCHMARK_WORKFLOW.md` | benchmark 目录与命名规范 |
|
| `BENCHMARK_WORKFLOW.md` | benchmark 目录与命名规范 |
|
||||||
| `scripts/SLO_STANDARDS.md` | 推理服务 SLO 标准(TTFT/TPOT) |
|
| `docs/SLO_STANDARDS.md` | 推理服务 SLO 标准(TTFT/TPOT) |
|
||||||
|
|
||||||
> 注:`.gitignore` 已排除 `envs/`、`deps/`、`tmp/`、`datasets/`、`logs/`、`*.zip`、原始 `raw_outputs/*.jsonl` 等大文件/中间文件。
|
> 注:`.gitignore` 已排除 `envs/`、`deps/`、`tmp/`、`datasets/`、`logs/`、`*.zip`、原始 `raw_outputs/*.jsonl` 等大文件/中间文件。
|
||||||
|
|
||||||
@ -99,7 +99,7 @@ python3 experiments/dsv4_h200_dspark/parse_results.py \
|
|||||||
|
|
||||||
## SLO 参考
|
## SLO 参考
|
||||||
|
|
||||||
详见 `scripts/SLO_STANDARDS.md`。
|
详见 `docs/SLO_STANDARDS.md`。
|
||||||
|
|
||||||
主要关注指标:
|
主要关注指标:
|
||||||
|
|
||||||
|
|||||||
@ -17,5 +17,5 @@
|
|||||||
|
|
||||||
## Notes
|
## Notes
|
||||||
|
|
||||||
- SLO check uses TTFT P95 and TPOT mean (the same criteria as `scripts/SLO_STANDARDS.md`).
|
- SLO check uses TTFT P95 and TPOT mean (the same criteria as `docs/SLO_STANDARDS.md`).
|
||||||
- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.
|
- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.
|
||||||
|
|||||||
@ -19,5 +19,5 @@
|
|||||||
|
|
||||||
## Notes
|
## Notes
|
||||||
|
|
||||||
- SLO check uses TTFT P95 and TPOT mean (the same criteria as `scripts/SLO_STANDARDS.md`).
|
- SLO check uses TTFT P95 and TPOT mean (the same criteria as `docs/SLO_STANDARDS.md`).
|
||||||
- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.
|
- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.
|
||||||
|
|||||||
@ -18,7 +18,7 @@ def load_result(result_root: Path) -> dict:
|
|||||||
|
|
||||||
|
|
||||||
def slo_status(ttft_p95_ms: float, tpot_mean_ms: float) -> str:
|
def slo_status(ttft_p95_ms: float, tpot_mean_ms: float) -> str:
|
||||||
# S2 tier targets from scripts/SLO_STANDARDS.md.
|
# S2 tier targets from docs/SLO_STANDARDS.md.
|
||||||
ttft_ok = ttft_p95_ms < 3000.0
|
ttft_ok = ttft_p95_ms < 3000.0
|
||||||
tpot_ok = tpot_mean_ms < 50.0
|
tpot_ok = tpot_mean_ms < 50.0
|
||||||
if ttft_ok and tpot_ok:
|
if ttft_ok and tpot_ok:
|
||||||
@ -75,7 +75,7 @@ def main():
|
|||||||
)
|
)
|
||||||
|
|
||||||
f.write("\n## Notes\n\n")
|
f.write("\n## Notes\n\n")
|
||||||
f.write("- SLO check uses TTFT P95 and TPOT mean (the same criteria as `scripts/SLO_STANDARDS.md`).\n")
|
f.write("- SLO check uses TTFT P95 and TPOT mean (the same criteria as `docs/SLO_STANDARDS.md`).\n")
|
||||||
f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n")
|
f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n")
|
||||||
|
|
||||||
print(f"Wrote comparison to {args.output}")
|
print(f"Wrote comparison to {args.output}")
|
||||||
|
|||||||
@ -0,0 +1,42 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"experiment": "dsv4_h200_long_context_matrix_sglang",
|
||||||
|
"run_id": "20260708-100016",
|
||||||
|
"timestamp": "2026-07-08T10:00:20+00:00",
|
||||||
|
"model": "/data/models/DeepSeek-V4-Flash",
|
||||||
|
"backend": "sglang",
|
||||||
|
"engine": "sglang",
|
||||||
|
"hardware": "8x NVIDIA H200 143GB",
|
||||||
|
"accelerator": "NVIDIA H200",
|
||||||
|
"chip": "nvidia_h200",
|
||||||
|
"script": "experiments/dsv4_h200_long_context_matrix/run_bench.sh",
|
||||||
|
"env": "/data/user1/yy/envs/sglang",
|
||||||
|
"git_commit": "7e35945",
|
||||||
|
"git_dirty": "dirty",
|
||||||
|
"description": "H200 long-context matrix sglang TP=8 for DeepSeek-V4-Flash"
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"tp": 8,
|
||||||
|
"cuda_visible_devices": "0,1,2,3,4,5,6,7",
|
||||||
|
"backend": "sglang",
|
||||||
|
"server_start_script": "experiments/dsv4_h200_long_context_matrix/start_sglang.sh",
|
||||||
|
"groups": [
|
||||||
|
{
|
||||||
|
"label": "64k",
|
||||||
|
"max_model_len": 80000,
|
||||||
|
"max_slots": 25
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "128k",
|
||||||
|
"max_model_len": 140000,
|
||||||
|
"max_slots": 10
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "256k",
|
||||||
|
"max_model_len": 270000,
|
||||||
|
"max_slots": 10
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"scenarios": []
|
||||||
|
}
|
||||||
@ -0,0 +1,42 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"experiment": "dsv4_h200_long_context_matrix_vllm",
|
||||||
|
"run_id": "20260708-100016",
|
||||||
|
"timestamp": "2026-07-08T10:00:20+00:00",
|
||||||
|
"model": "/data/models/DeepSeek-V4-Flash",
|
||||||
|
"backend": "vllm",
|
||||||
|
"engine": "vllm",
|
||||||
|
"hardware": "8x NVIDIA H200 143GB",
|
||||||
|
"accelerator": "NVIDIA H200",
|
||||||
|
"chip": "nvidia_h200",
|
||||||
|
"script": "experiments/dsv4_h200_long_context_matrix/run_bench.sh",
|
||||||
|
"env": "/data/user1/yy/envs/vllm",
|
||||||
|
"git_commit": "7e35945",
|
||||||
|
"git_dirty": "dirty",
|
||||||
|
"description": "H200 long-context matrix vllm TP=8 for DeepSeek-V4-Flash"
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"tp": 8,
|
||||||
|
"cuda_visible_devices": "0,1,2,3,4,5,6,7",
|
||||||
|
"backend": "vllm",
|
||||||
|
"server_start_script": "experiments/dsv4_h200_long_context_matrix/start_vllm.sh",
|
||||||
|
"groups": [
|
||||||
|
{
|
||||||
|
"label": "64k",
|
||||||
|
"max_model_len": 80000,
|
||||||
|
"max_slots": 25
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "128k",
|
||||||
|
"max_model_len": 140000,
|
||||||
|
"max_slots": 10
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "256k",
|
||||||
|
"max_model_len": 270000,
|
||||||
|
"max_slots": 10
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"scenarios": []
|
||||||
|
}
|
||||||
@ -37,5 +37,5 @@
|
|||||||
|
|
||||||
## Notes
|
## Notes
|
||||||
|
|
||||||
- SLO check uses TTFT P95 and TPOT mean (the same criteria as `scripts/SLO_STANDARDS.md`).
|
- SLO check uses TTFT P95 and TPOT mean (the same criteria as `docs/SLO_STANDARDS.md`).
|
||||||
- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.
|
- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user