From 63ab41b65ac394fe1ab3078761bff0d7702a56bf Mon Sep 17 00:00:00 2001 From: shishi Date: Wed, 29 Jul 2026 11:51:24 +0800 Subject: [PATCH] docs: consolidate project docs (dedup, relocate, expand 910C client guide) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Project-level documentation was scattered and duplicated across README.md, BENCHMARK_WORKFLOW.md, and docs/EXPERIMENT_GUIDE.md (directory layout + scripts/common component table repeated 3x). Reorganize into a clear single-source-of-truth structure. Changes: - README.md: drop the 6 stale changelog entries at the top (latest was 07-21; history lives in git log). Replace the duplicated directory- layout + scripts/common sections with a one-line link to docs/EXPERIMENT_GUIDE.md. (151 -> 99 lines) - BENCHMARK_WORKFLOW.md -> docs/BENCHMARK_WORKFLOW.md: relocate into docs/. Replace its duplicated Directory Layout and Quick Start/Adding sections with links to EXPERIMENT_GUIDE / README / NEW_PLATFORM_GUIDE; keep the unique parts (Rules, Naming Conventions, Final JSON Schema, Checklist). (394 -> 224 lines) - docs/EXPERIMENT_GUIDE.md: now the single authority for directory layout + component table + experiment conventions. Add a cross-link from the results.json field list to BENCHMARK_WORKFLOW's full JSON Schema and Naming Conventions. - docs/H200_QUICKSTART.md: deleted (outdated, repeatedly references removed legacy scripts; H200 usage is covered by ADAPTIVE_CONCURRENCY_USAGE and experiment READMEs). - docs/DSV4_INFERENCE_COMPARISON_REPORT.md -> experiments/h200/ dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/: this is an experiment report, not a project doc; relocate next to its sibling report.md. - envs/ASCEND_910C_ENV_SETUP.md §8: expand the vague "pip install sglang" note into a full sglang client image build guide -- pin sglang 0.5.2 (not latest; >=0.5.16 deprecates bench_serving and breaks the parser), --no-deps minimal install loop, docker commit to a local image, with the exact commands used to build local/vllm-ascend:0.23-a3-dsv4-sglang. - experiments/h200/dsv4_h200_vllm_tp2_custom_bench/README.md: fix the now-broken link to BENCHMARK_WORKFLOW.md (../../ -> ../../../docs/). - .gitignore: ignore *.bak.glm52orig scratch backups. Also includes the add16 adaptive_results produced by the dsv4 TP=4/DP=2 runs on 910c.1. --- .gitignore | 3 + README.md | 57 +----- .../BENCHMARK_WORKFLOW.md | 182 +----------------- docs/EXPERIMENT_GUIDE.md | 2 + docs/H200_QUICKSTART.md | 99 ---------- envs/ASCEND_910C_ENV_SETUP.md | 57 +++++- .../adaptive_points.jsonl | 0 .../adaptive_shapes.jsonl | 0 .../adaptive_summary.jsonl | 0 .../adaptive_summary.md | 6 + .../run_manifest.json | 27 +++ .../adaptive_20260728-165926/shapes.tsv | 31 +++ .../adaptive_points.jsonl | 0 .../adaptive_shapes.jsonl | 0 .../adaptive_summary.jsonl | 0 .../adaptive_summary.md | 6 + .../run_manifest.json | 27 +++ .../adaptive_20260729-110354/shapes.tsv | 31 +++ .../adaptive_points.jsonl | 0 .../adaptive_shapes.jsonl | 0 .../adaptive_summary.jsonl | 0 .../adaptive_summary.md | 6 + .../run_manifest.json | 27 +++ .../adaptive_20260729-112009/shapes.tsv | 31 +++ .../adaptive_points.jsonl | 0 .../adaptive_shapes.jsonl | 0 .../adaptive_summary.jsonl | 0 .../adaptive_summary.md | 6 + .../run_manifest.json | 27 +++ .../adaptive_20260729-113439/shapes.tsv | 31 +++ .../adaptive_points.jsonl | 3 + .../adaptive_shapes.jsonl | 1 + .../run_manifest.json | 27 +++ .../adaptive_20260729-113709/shapes.tsv | 31 +++ .../DSV4_INFERENCE_COMPARISON_REPORT.md | 0 .../dsv4_h200_vllm_tp2_custom_bench/README.md | 2 +- 36 files changed, 383 insertions(+), 337 deletions(-) rename BENCHMARK_WORKFLOW.md => docs/BENCHMARK_WORKFLOW.md (60%) delete mode 100644 docs/H200_QUICKSTART.md create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/adaptive_points.jsonl create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/adaptive_shapes.jsonl create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/adaptive_summary.jsonl create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/adaptive_summary.md create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/run_manifest.json create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/shapes.tsv create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/adaptive_points.jsonl create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/adaptive_shapes.jsonl create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/adaptive_summary.jsonl create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/adaptive_summary.md create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/run_manifest.json create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/shapes.tsv create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/adaptive_points.jsonl create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/adaptive_shapes.jsonl create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/adaptive_summary.jsonl create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/adaptive_summary.md create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/run_manifest.json create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/shapes.tsv create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/adaptive_points.jsonl create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/adaptive_shapes.jsonl create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/adaptive_summary.jsonl create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/adaptive_summary.md create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/run_manifest.json create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/shapes.tsv create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/adaptive_points.jsonl create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/adaptive_shapes.jsonl create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/run_manifest.json create mode 100644 experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/shapes.tsv rename {docs => experiments/h200/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349}/DSV4_INFERENCE_COMPARISON_REPORT.md (100%) diff --git a/.gitignore b/.gitignore index 2fc1497..be8ff8a 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,6 @@ envs/* # Tooling artifacts skills-lock.json + +# 临时备份文件 +*.bak.glm52orig diff --git a/README.md b/README.md index d382785..35b5189 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,6 @@ # sskj — 多平台大模型推理性能基准测试项目 -> **更新（2026-07-21 14:12:30 +0800）** -> - RTX 6000D / DeepSeek-V4-Flash 的 SGLang 全量与 tiny 脚本切换到 `lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45`，MoE backend 保持 `auto`，由 nightly 自动选择可用的 FlashInfer MoE 实现；同时使用独立 nightly CUDA 13 cache，避免复用旧 SM120 修补镜像的编译缓存。 - -> **更新（2026-07-21 14:00:30 +0800）** -> - adaptive 并发搜索新增可配置的 `C=1 OOM` 配置级短路：同一 TP/DP 在最小并发仍 OOM 时，直接跳过该组更长输入，避免把已经确定不可部署的长度继续压测；6000D DSV4-Flash SGLang tiny 已启用，并支持从已有结果恢复时识别此前的该边界。 - -> **更新（2026-07-21 11:54:30 +0800）** -> - 新增 RTX 6000D / DeepSeek-V4-Flash tiny 自适应并发实验：输出固定 1K，输入遍历 1K、2K、4K、8K、16K、32K、64K、128K，并发采用 `C=16` 起、每次加 16、首点失败时按 `16 -> 8 -> 1` 回退。 -> - vLLM 覆盖 TP2/DP4、TP4/DP2、TP8/DP1，部署固定为 128K context、128 活跃请求、0.9 显存比例；SGLang 覆盖 TP4/DP2、TP8/DP1，固定为 128K、64、0.9，TP2/DP4 因 SM120 Marlin 权重加载 OOM 明确排除。 - -> **更新（2026-07-20 17:55:20 +0800）** -> - H20 SGLang TPxDP matrix 默认传入 `--cuda-graph-max-bs-decode 128`，使高并发 decode 在 batch 不超过 128 时持续使用 CUDA Graph；Docker 与 native 启动入口均已覆盖。 - -> **更新（2026-07-20 17:25:40 +0800）** -> - RTX 6000D 的 SGLang DSV4-Flash 默认传入 `--cuda-graph-max-bs-decode 16`，避免 SM120 sparse MLA 在大 batch CUDA Graph 预捕获时 OOM。该参数仅限制预捕获图的 decode batch 上限，不限制服务的 `max-running-requests` 或 benchmark 并发；其他平台默认不受影响。 -> - H20、RTX 6000D、P800 的配置不再硬编码 sskj 仓库绝对路径：仓库内的环境、数据集、缓存和 P800 patch 均通过共享的 `${ROOT_DIR}` 推导，仓库迁移后无需逐个修改路径。 - -> **更新（2026-07-20，PR 自动化测试）** -> - 仅用于验证 Gitee 受保护分支的自动 Pull Request 创建与评审流程；不修改实验代码、配置或结果口径。 - -> **更新（2026-07-20，`903084c`）** -> - RTX 6000D 的 SGLang DSV4-Flash 默认传入 `--cuda-graph-max-bs-decode 16`，避免 SM120 sparse MLA 在大 batch CUDA Graph 预捕获时 OOM。该参数仅限制预捕获图的 decode batch 上限，不限制服务的 `max-running-requests` 或 benchmark 并发；其他平台默认不受影响。 -> - H20 / RTX 6000D 的 TPxDP matrix baseline 不再显式限制模型上下文或服务内并发：vLLM 使用默认 `max-model-len` / `max-num-seqs`，SGLang 使用默认 `context-length` / `max-running-requests`。 -> - SGLang 固定使用 `--moe-runner-backend marlin`；vLLM 保持框架默认 MoE backend。 -> - `run_adaptive_concurrency_add16.sh` 在首点 OOM 时会重启服务并按 `C=16 -> 8 -> 1` 回退；仅 H20 / RTX 6000D matrix 默认开启，H200 / P800 保持原行为。 -> - RTX 6000D 旧 SGLang 结果使用 262K context cap，超过该范围或 C>32 的点不可与新 baseline 混用；请使用新的 `RUN_ID` 重跑受影响场景。 - +> 历史更新见 `git log`。项目目的与工作流见下方。 > **项目目的**：当新显卡（GPU/NPU）到货时，用最短时间完成大模型在该平台上的推理性能评估与部署配置选型。 > 当前模型：DeepSeek-V4-Flash（FP8 / INT8）；后续接入 GLM5.2，**完全复用**本项目的实验与报告流程。 > 新平台接入 SOP：[`docs/NEW_PLATFORM_GUIDE.md`](docs/NEW_PLATFORM_GUIDE.md)。 @@ -55,34 +29,7 @@ ## 目录结构 -``` -platforms/ # 平台配置（每平台一个 .env）+ 平台补丁（patches/） -scripts/common/ # 跨实验复用的编排组件（见下） -experiments/ # 实验目录（三层结构） -├── TEMPLATE/ # 老式固定场景对比实验模板（sglang vs vllm） -├── h20/ # NVIDIA H20 96GB -├── h200/ # NVIDIA H200 143GB -├── p800/ # Kunlun P800 XPU -└── pro6000/ # NVIDIA RTX 6000D -docs/ # 规范与指南（SLO、实验规范、新卡接入） -envs/ # 环境搭建文档（具体 venv 不提交，见 .gitignore） -datasets/ # benchmark 数据集（不提交） -BENCHMARK_WORKFLOW.md # 结果目录与命名规范、results.json schema -``` - -### scripts/common/ 组件 - -| 文件 | 职责 | -|---|---| -| `lib.sh` | 日志、health check、metadata JSON 生成 | -| `platform.sh` | 平台自动探测并加载 `platforms/.env` | -| `adaptive_bench_lib.sh` | **自适应并发搜索核心库**（引擎无关，靠 5 个 `engine_*` 回调注入平台逻辑） | -| `adaptive_concurrency.py` | 上者的 Python 助手（shapes 生成、结果解析、汇总） | -| `parse_backend.py` | 固定并发实验：raw jsonl → results.json + report.md | -| `compare.py` | SGLang vs vLLM 横向对比表（老式实验用） | -| `warmup.py` | 服务预热 | -| `server_docker.sh` / `bench_client_docker.sh` | P800 专用 Docker 服务端/客户端生命周期 | -| `adaptive_heartbeat.sh` | 长跑实验的心跳监控（手动独立运行） | +仓库的目录布局、`scripts/common/` 组件职责、结果命名规范与 `results.json` schema，统一见 [`docs/EXPERIMENT_GUIDE.md`](docs/EXPERIMENT_GUIDE.md)（单一权威来源）与 [`docs/BENCHMARK_WORKFLOW.md`](docs/BENCHMARK_WORKFLOW.md)。 ## 实验索引 diff --git a/BENCHMARK_WORKFLOW.md b/docs/BENCHMARK_WORKFLOW.md similarity index 60% rename from BENCHMARK_WORKFLOW.md rename to docs/BENCHMARK_WORKFLOW.md index b49e522..c37b64d 100644 --- a/BENCHMARK_WORKFLOW.md +++ b/docs/BENCHMARK_WORKFLOW.md @@ -2,37 +2,7 @@ ## Directory Layout -``` -/ # 本仓库根目录（如 /data1/yy/sskj，按机器调整） -├── platforms/ # chip/accelerator platform configs -│ ├── kunlun_p800.env -│ ├── nvidia_h20.env -│ ├── nvidia_h200.env -│ ├── nvidia_rtx6000d.env -│ └── patches/kunlun_p800/ # runtime patches required by some images -├── scripts/ -│ └── common/ # reusable components (lib.sh, platform.sh, parse_backend.py, adaptive_bench_lib.sh, ...) -├── docs/ -│ └── SLO_STANDARDS.md # 推理服务 SLO 标准 -├── experiments/ # experiment-centric directories (preferred): experiments/// -│ ├── TEMPLATE/ # template for fixed-scenario sglang-vs-vllm experiments -│ └── / # h20 / h200 / p800 / pro6000 -│ └── dsv4_p800_sglang/ -│ ├── README.md -│ ├── config.env # experiment-level configuration -│ ├── start_server.sh -│ ├── run_bench.sh -│ ├── parse_results.py -│ └── results/ -│ └── 20260708-XXXXXX/ -│ ├── report.md -│ ├── results.json -│ └── logs/ -├── datasets/ # benchmark datasets -└── envs/ # Python virtual environments -``` - -注：legacy 的 `scripts/benchmark_dspark_0707/` 已删除；顶层 `bench_results/` 与 `logs/` 已不在仓库中（历史结果已归档迁移至仓库外）。 +仓库目录结构、`scripts/common/` 组件职责见 [`EXPERIMENT_GUIDE.md`](EXPERIMENT_GUIDE.md) §1（单一权威来源，避免多处维护漂移）。 ## Rules @@ -230,152 +200,12 @@ The JSON file inside each experiment run directory (`experiments// -``` - -## Adding a New Platform or Experiment - -### Quick Start: New Experiment on an Existing Platform (e.g. a new P800 box) - -Use this when the platform already exists (`platforms/.env` present) and you only want to test a new model/config on a machine. - -**0. Machine setup (one-time, only on a brand-new machine)** - -```bash -git clone https://git.meta-stone.net/qqtang/sskj.git /data1/yy/sskj -cd /data1/yy/sskj -git config credential.helper store # push once to save the token -# Verify platform auto-detection: -bash -c 'source scripts/common/platform.sh && echo "$PLATFORM | $HARDWARE | $ENGINE"' -# Ensure platform deps exist (see platforms/.env): docker image pulled, MODEL_ROOT has the model, datasets/ present. -``` - -If the platform is not auto-detected, run with `PLATFORM= ...`. Platform-wide paths/images live in `platforms/.env` - edit that file (not the experiment) if a path differs on this machine. - -**1. Create the experiment from TEMPLATE** - -```bash -cd /data1/yy/sskj -cp -r experiments/TEMPLATE experiments/p800/ # e.g. qwen3_8b_p800_sglang_tp4 -cd experiments/p800/ -``` - -**2. Edit `config.env`** - the single source of experiment params (model, TP, ports, scenarios). All values use `${VAR:-default}`, so they can be overridden by env vars at run time. - -**3. Edit `start_sglang.sh` / `start_server.sh`** - the deployment script (`docker run` + launch flags). This is where you change deployment params (TP, mem-fraction, attention backend, cuda graph, ...). Reference `config.env` variables (`$TP`, `$MEM_FRACTION_STATIC`, ...) rather than hardcoding. - -**4. Run** - -```bash -bash run_bench.sh # auto-detects platform -# or with overrides / explicit RUN_ID: -PLATFORM=kunlun_p800 RUN_ID=tp4_20260722 bash run_bench.sh -# quick config probe without editing files: -TP=4 MEM_FRACTION_STATIC=0.85 MAX_RUNNING=32 bash run_bench.sh -``` - -Results land in `results//` (`report.md`, `results.json`; `raw_outputs/` and `logs/` are gitignored). Watch `logs/orchestrator.log` and `logs/server.log`. - -**5. Commit code + final artifacts, push, open PR** - -```bash -cd /data1/yy/sskj -git checkout -b feat/p800- -git add experiments/p800//config.env \ - experiments/p800//start_sglang.sh \ - experiments/p800//run_bench.sh \ - experiments/p800//README.md \ - experiments/p800//results//report.md \ - experiments/p800//results//results.json -git commit -m "feat(p800): add experiment" -git push -u origin feat/p800- -``` - -Do NOT `git add` `raw_outputs/` or `logs/` (already ignored). Run through the [Checklist](#checklist-before-committing--archiving) below before pushing. - -> Adding a brand-new chip (not just a new experiment on an existing platform)? Follow steps 1-4 below instead. - - -### 1. Add or update a platform config - -Create `platforms/.env` with identity and platform-wide paths: - -```bash -CHIP="my_chip" -ACCELERATOR="My Accelerator" -HARDWARE="8x My Accelerator" -ENGINE="vllm-myengine" -DEFAULT_PORT="30000" -MODEL_ROOT="/data/models" -``` - -For Docker-based platforms, also set `DOCKER_IMAGE`, `CONTAINER_NAME`, -`CONTAINER_PYTHON`, and `PATCH_ROOT` (see `platforms/kunlun_p800.env`). -For host-native platforms, set the relevant venv paths (see -`platforms/nvidia_h200.env`). - -### 2. Create an experiment directory - -``` -experiments/// -├── README.md # Purpose and usage -├── config.env # Model, port, scenarios, engine overrides -├── start_server.sh # (optional) platform-specific server launch -├── run_bench.sh # Orchestrator -└── parse_results.py # Convert raw outputs to results.json + report.md -``` - -At minimum, `run_bench.sh` should: - -1. Source `scripts/common/lib.sh` and `scripts/common/platform.sh`. -2. Read `config.env`. -3. Create `experiments///results//`. -4. Call `write_metadata_json` to create `results.json`. -5. Record the exact server launch command/args in `results.json` `config.server_args` (or `phaseN_server_args`). -6. Run the benchmark scenarios. -7. Call `parse_results.py` to generate `report.md`. - -### 3. Reuse shared helpers - -- `scripts/common/lib.sh`: logging, health checks, metadata JSON. -- `scripts/common/platform.sh`: platform auto-detection and env loading. -- `scripts/common/warmup.py`: server 预热。 -- `scripts/common/parse_backend.py`: raw jsonl -> `results.json` + `report.md`。 -- `scripts/common/compare.py`: SGLang vs vLLM 横向对比表。 - -Docker/XPU 平台可以在 `scripts/common/` 下新增自己的 helper，但不要在实验目录复制通用逻辑。 - -### 4. Example experiments - -- Docker / XPU: `experiments/p800/dsv4_p800_sglang/` -- Native / H200: `experiments/h200/dsv4_h200_dspark/` - -See `docs/H200_QUICKSTART.md` for a concrete H200 porting walkthrough. +快速复现、新增实验、新平台接入的步骤见： +- [`../README.md`](../README.md) §快速复现（命令示例） +- [`EXPERIMENT_GUIDE.md`](EXPERIMENT_GUIDE.md) §2 新增实验（config.env 必备字段） +- [`NEW_PLATFORM_GUIDE.md`](NEW_PLATFORM_GUIDE.md)（新平台接入 SOP） ## Checklist Before Committing / Archiving diff --git a/docs/EXPERIMENT_GUIDE.md b/docs/EXPERIMENT_GUIDE.md index 2a6b2bd..d8729de 100644 --- a/docs/EXPERIMENT_GUIDE.md +++ b/docs/EXPERIMENT_GUIDE.md @@ -93,6 +93,8 @@ VLLM_START_SCRIPT="${SCRIPT_DIR:-.}/start_vllm.sh" 这些通过 `scripts/common/lib.sh` 中的 `write_metadata_json` 和 `jq` 注入。 +> 完整的 `results.json` schema（含 scenarios/latencies/slo_status 等字段）见 [`BENCHMARK_WORKFLOW.md`](BENCHMARK_WORKFLOW.md) §Final JSON Schema；结果目录命名规范见同文 §Naming Conventions。 + ## 3. 平台与硬件适配 ### 3.1 芯片级默认配置 diff --git a/docs/H200_QUICKSTART.md b/docs/H200_QUICKSTART.md deleted file mode 100644 index 41b89e7..0000000 --- a/docs/H200_QUICKSTART.md +++ /dev/null @@ -1,99 +0,0 @@ -# H200 Quick Start Guide - -This guide covers how to use this benchmark repository on an NVIDIA H200 machine. - -## Current state - -The older H200 experiments (e.g. `experiments/h200/dsv4_h200_dspark/`) are based on **native host virtual environments** (not Docker): - -- Server engine: `vllm-dspark` -- Server env: `envs/vllm-dspark` -- Benchmark client env: `envs/sglang` (uses `sglang.bench_serving --backend vllm`) -- Default model: `/data/models/DeepSeek-V4-Flash-DSpark` -- Default port: `30004` - -Note: the current mainstream H200 experiments — the TP/DP matrix adaptive-concurrency suites `experiments/h200/dsv4_h200_vllm_tp_dp_matrix/` and `experiments/h200/dsv4_h200_sglang_tp_dp_matrix/` — are Docker-based (`USE_DOCKER=1` for the server, benchmark client via `DOCKER_CLIENT_IMAGE`); see `experiments/ADAPTIVE_CONCURRENCY_USAGE.md`. - -The legacy benchmark suite `scripts/benchmark_dspark_0707/` has been removed from this repo. The migration wrapper experiment `experiments/h200/dsv4_h200_dspark/` still exists and produces the `experiments/h200//results//` layout, but its default `LEGACY_GRID_SCRIPT` / `SERVER_START_SCRIPT` point at the removed scripts and must be overridden to run. - -## 1. Pull and verify - -```bash -git clone sskj # clone path: adjust to your machine -cd sskj - -# Platform should auto-detect as nvidia_h200 -source scripts/common/platform.sh -``` - -If auto-detection fails, set it explicitly: - -```bash -PLATFORM=nvidia_h200 source scripts/common/platform.sh -``` - -## 2. Legacy DSpark benchmark grid (removed) - -The legacy suite `scripts/benchmark_dspark_0707/` has been removed from this repo, along with its `bench_results/dspark_grid_/` output location (historical results were archived elsewhere). Use the wrapper experiment below, or the current tp_dp matrix experiments (`experiments/h200/dsv4_h200_{vllm,sglang}_tp_dp_matrix/`), instead. - -## 3. Run the migration wrapper experiment - -This produces results in the experiment-centric layout: - -```bash -bash experiments/h200/dsv4_h200_dspark/run_bench.sh -``` - -Results land in `experiments/h200/dsv4_h200_dspark/results//`. - -Caveat: the wrapper's default `LEGACY_GRID_SCRIPT` and `SERVER_START_SCRIPT` refer to the removed `scripts/benchmark_dspark_0707/` suite and `scripts/start_dsv4_dspark_8card.sh`; set both explicitly (or restore equivalent scripts) before running. - -## 4. Create a new H200 experiment - -To add a new H200 benchmark (for example a different model or engine), create: - -``` -experiments/h200// -├── README.md # What this experiment measures -├── config.env # Model, port, scenarios, venv paths -├── start_server.sh # (optional) native server launch -├── run_bench.sh # Orchestrator: server → benchmark → stop -└── parse_results.py # Generate results.json + report.md -``` - -Minimum `config.env`: - -```bash -EXPERIMENT="${EXPERIMENT:-}" -MODEL_NAME="${MODEL_NAME:-DeepSeek-V4-Flash-DSpark}" -MODEL_PATH="${MODEL_PATH:-/data/models/DeepSeek-V4-Flash-DSpark}" -PORT="${PORT:-30004}" -BACKEND="${BACKEND:-vllm}" -ENGINE="${ENGINE:-vllm-dspark}" - -# Native venvs (paths are per-machine; adjust to yours) -VENV_SERVER="${VENV_SERVER:-/path/to/envs/vllm-dspark}" -VENV_CLIENT="${VENV_CLIENT:-/path/to/envs/sglang}" - -SCENARIOS=( - "32 512 256" -) -``` - -Then source the platform loader and shared helpers at the top of `run_bench.sh`: - -```bash -source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" -source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" -``` - -For a concrete example, see `experiments/h200/dsv4_h200_dspark/`. For the current mainstream form (TP/DP matrix + adaptive concurrency search, Docker-based), copy `experiments/h200/dsv4_h200_vllm_tp_dp_matrix/` instead; usage in `experiments/ADAPTIVE_CONCURRENCY_USAGE.md`. - -## 5. Cross-platform comparison - -跨平台对比脚本目前未统一提供。可分别读取各实验 `results//results.json` 中的结构化数据，按 scenario 聚合后生成对比表。 - -## Notes - -- `platforms/nvidia_h200.env` still carries venv defaults under `/data/user1/yy` (paths from another machine). Override `VENV_VLLM_DSPARK`, `VENV_SGLANG`, `MODEL_ROOT`, or `SERVER_START_SCRIPT` to match your machine. -- Native server management helpers are not as mature as the Docker helpers in `scripts/common/server_docker.sh`. The wrapper's default `SERVER_START_SCRIPT` (`scripts/start_dsv4_dspark_8card.sh`) no longer exists in this repo and must be overridden. diff --git a/envs/ASCEND_910C_ENV_SETUP.md b/envs/ASCEND_910C_ENV_SETUP.md index fc556bb..cb4c1de 100644 --- a/envs/ASCEND_910C_ENV_SETUP.md +++ b/envs/ASCEND_910C_ENV_SETUP.md @@ -127,18 +127,63 @@ bash run_adaptive_concurrency_add16.sh ## 8. Benchmark Client 说明 ⚠️ -sskj 的压测客户端是 `sglang.bench_serving`，但 **vllm-ascend 官方镜像不含 sglang**。910C 上有两个选择： +sskj 的压测客户端是 `sglang.bench_serving`，但 **vllm-ascend 官方镜像不含 sglang**。两个选择： -1. **(推荐) 容器内装 sglang**：进容器 `pip install sglang`（或 `sglang[all]`），之后 `docker exec` 跑客户端。需要把 sglang 装进镜像或每次启动后手动装。 -2. **外部 sglang 镜像**：设 `USE_DOCKER_CLIENT=1`，提供 `DOCKER_CLIENT_IMAGE=lmsysorg/sglang:xxx`，用独立容器通过 host 网络打 vLLM 的 OpenAI API。但 sglang 官方镜像多为 x86 + CUDA，aarch64 NPU 节点上可能拉不到对应架构镜像。 +1. **(推荐) 构建带 sglang 的 vllm-ascend 镜像**：见 §8.1，一次性构建后所有实验复用。 +2. **外部 sglang 镜像**：设 `USE_DOCKER_CLIENT=1`，提供 `DOCKER_CLIENT_IMAGE=lmsysorg/sglang:xxx`，用独立容器通过 host 网络打 vLLM 的 OpenAI API。但 sglang 官方镜像多为 x86 + CUDA，aarch64 NPU 节点上拉不到对应架构镜像，不推荐。 -建议冒烟前先确认客户端方案，否则 adaptive 搜索会在 `engine_run_bench` 阶段失败。验证命令： +建议冒烟前先确认客户端方案，否则 adaptive 搜索会在 `engine_run_bench` 阶段失败。 + +### 8.1 构建带 sglang 客户端的 vllm-ascend 镜像 + +> **版本要求**：必须装 **sglang 0.5.2**，不是最新版。 +> - sglang ≥ 0.5.16 已废弃 `sglang.bench_serving`（改为 `sglang.benchmark.serving`），输出格式变了，与 `scripts/common/adaptive_concurrency.py` / `parse_backend.py` 的解析逻辑不兼容。 +> - glm52 实验的解析器修复（commit 98cdb67）针对的就是 0.5.2 的输出格式。dsv4/glm52 实验的 `config.env` 默认镜像 `local/vllm-ascend:0.23-a3-*-sglang` 即基于 0.5.2 构建。 + +**关键原则**：用 `pip install --no-deps` 只装 sglang 本体 + bench_serving 的轻量依赖，**不要装完整 sglang**（会拉 torch/transformers 等重依赖，破坏容器内 vllm 环境）。 + +构建步骤（在 910C 节点上，需 sudo docker）： ```bash -# 进容器看是否有 sglang -docker run --rm python -c "import sglang; print(sglang.__version__)" +# 1. 启动一个临时容器（基础镜像 = 实验用的 vllm-ascend 镜像） +BASE=quay.io/ascend/vllm-ascend:v0.23.0rc1-a3-openeuler +sudo docker rm -f sglang-build 2>/dev/null +sudo docker run -d --name sglang-build \ + --device /dev/davinci0 --device /dev/davinci_manager \ + --device /dev/devmm_svm --device /dev/hisi_hdc \ + -v /usr/local/dcmi:/usr/local/dcmi \ + -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \ + -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \ + -v /etc/ascend_install.info:/etc/ascend_install.info \ + $BASE sleep infinity + +# 2. 装 sglang 0.5.2 本体（--no-deps，不碰 vllm 环境） +sudo docker exec sglang-build pip install --no-deps "sglang==0.5.2" + +# 3. 循环补齐 bench_serving 缺失的轻量依赖（自动检测 ModuleNotFoundError 并安装） +# 会装约 8 个包：ipython/traitlets/stack_data/executing/asttokens/pure_eval/prompt_toolkit/wcwidth +# （都是 IPython 依赖链，bench_serving 通过 sglang.utils 间接引入） +sudo docker exec sglang-build bash -c ' +for i in $(seq 1 50); do + OUT=$(python -c "import sglang.bench_serving" 2>&1) + [ -z "$OUT" ] && { echo "IMPORT_OK"; break; } + MOD=$(echo "$OUT" | grep -oE "No module name .[a-zA-Z0-9_]+." | head -1 | sed "s/No module name //;s/.//") + [ -z "$MOD" ] && { echo "NON_MODULE_ERR: $OUT"; break; } + echo "iter $i: $MOD" + pip install --no-deps "$MOD" 2>&1 | grep -i Successfully +done +' + +# 4. 验证 +sudo docker exec sglang-build python -m sglang.bench_serving --help | head -3 + +# 5. commit 成新镜像（tag 按实验命名，如 dsv4 / glm52） +sudo docker commit sglang-build local/vllm-ascend:0.23-a3-dsv4-sglang +sudo docker rm -f sglang-build ``` +构建完成后，把 `config.env` 的 `DOCKER_IMAGE` 指向新镜像即可。容器内 Python 路径是 `/usr/local/python3.12.13/bin/python3`（不是 `/usr/local/bin/python`，后者不存在），`config.env` 的 `CONTAINER_PYTHON` 已设为此值。 + ## 9. NPU 监控 公共库 `adaptive_bench_lib.sh` 的 GPU 监控写死 `nvidia-smi`，910C 实验脚本已用 `npu-smi info` 重写 `adaptive_start_gpu_monitor` / `start_gpu_monitor`，输出与 nvidia-smi 相同的 CSV 列（timestamp, index, memory.used, memory.total, utilization.gpu），下游 `parse_backend.py` 无需改动。 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/adaptive_points.jsonl b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/adaptive_points.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/adaptive_shapes.jsonl b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/adaptive_shapes.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/adaptive_summary.jsonl b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/adaptive_summary.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/adaptive_summary.md b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/adaptive_summary.md new file mode 100644 index 0000000..1ba5678 --- /dev/null +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/adaptive_summary.md @@ -0,0 +1,6 @@ +# Adaptive concurrency search summary + +| Engine | TP | DP | ISL | OSL | Stop | Saturation C | Best TPS C | Best Total TPS | Max successful C | +|---|---:|---:|---:|---:|---|---:|---:|---:|---:| + +`Saturation C` is the first point in the final low-gain streak. `Best TPS C` is the tested point with the highest observed Total TPS. diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/run_manifest.json b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/run_manifest.json new file mode 100644 index 0000000..e34d8aa --- /dev/null +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/run_manifest.json @@ -0,0 +1,27 @@ +{ + "experiment": "dsv4_910c_vllm_tp_dp_matrix", + "engine": "vllm", + "run_id": "adaptive_20260728-165926", + "model": "/mnt/models/DeepSeek-V4-Flash-w8a8-mtp", + "hardware": "8x Ascend910 (16 dies, 64GB HBM/die)", + "matrix": "/mnt/yy/sskj/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/matrix.json", + "dataset": "random", + "tokenize_prompt": false, + "random_range_ratio": 1.0, + "search": { + "start_concurrency": 16, + "max_concurrency": 256, + "multiplier": 2, + "initial_backoff_concurrencies": [ + 8, + 1 + ], + "num_prompts_multiplier": 5, + "min_tps_gain_pct": 2.0, + "plateau_patience": 2, + "warmup_max_requests": 0, + "ttft_slo_ms": 4000.0, + "enable_ttft_slo_stop": 1, + "ttft_group_skip_ms": 8000.0 + } +} diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/shapes.tsv b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/shapes.tsv new file mode 100644 index 0000000..35b6705 --- /dev/null +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260728-165926/shapes.tsv @@ -0,0 +1,31 @@ +mark input_len output_len +Y 1024 128 +Y 1024 256 +Y 1024 512 +Y 1024 1024 +Y 1024 2048 +Y 1024 4096 +Y 4096 128 +Y 4096 256 +Y 4096 512 +Y 4096 1024 +Y 4096 2048 +Y 4096 4096 +Y 8192 128 +Y 8192 256 +Y 8192 512 +Y 8192 1024 +Y 8192 2048 +Y 8192 4096 +Y 16384 128 +Y 16384 256 +Y 16384 512 +Y 16384 1024 +Y 16384 2048 +Y 32768 128 +Y 32768 256 +Y 32768 512 +Y 32768 1024 +Y 65536 128 +Y 65536 256 +Y 131072 128 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/adaptive_points.jsonl b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/adaptive_points.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/adaptive_shapes.jsonl b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/adaptive_shapes.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/adaptive_summary.jsonl b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/adaptive_summary.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/adaptive_summary.md b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/adaptive_summary.md new file mode 100644 index 0000000..1ba5678 --- /dev/null +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/adaptive_summary.md @@ -0,0 +1,6 @@ +# Adaptive concurrency search summary + +| Engine | TP | DP | ISL | OSL | Stop | Saturation C | Best TPS C | Best Total TPS | Max successful C | +|---|---:|---:|---:|---:|---|---:|---:|---:|---:| + +`Saturation C` is the first point in the final low-gain streak. `Best TPS C` is the tested point with the highest observed Total TPS. diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/run_manifest.json b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/run_manifest.json new file mode 100644 index 0000000..003cfe8 --- /dev/null +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/run_manifest.json @@ -0,0 +1,27 @@ +{ + "experiment": "dsv4_910c_vllm_tp_dp_matrix", + "engine": "vllm", + "run_id": "adaptive_20260729-110354", + "model": "/mnt/models/DeepSeek-V4-Flash-w8a8-mtp", + "hardware": "8x Ascend910 (16 dies, 64GB HBM/die)", + "matrix": "/mnt/yy/sskj/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/matrix.json", + "dataset": "random", + "tokenize_prompt": false, + "random_range_ratio": 1.0, + "search": { + "start_concurrency": 16, + "max_concurrency": 256, + "multiplier": 2, + "initial_backoff_concurrencies": [ + 8, + 1 + ], + "num_prompts_multiplier": 5, + "min_tps_gain_pct": 2.0, + "plateau_patience": 2, + "warmup_max_requests": 0, + "ttft_slo_ms": 4000.0, + "enable_ttft_slo_stop": 1, + "ttft_group_skip_ms": 8000.0 + } +} diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/shapes.tsv b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/shapes.tsv new file mode 100644 index 0000000..35b6705 --- /dev/null +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-110354/shapes.tsv @@ -0,0 +1,31 @@ +mark input_len output_len +Y 1024 128 +Y 1024 256 +Y 1024 512 +Y 1024 1024 +Y 1024 2048 +Y 1024 4096 +Y 4096 128 +Y 4096 256 +Y 4096 512 +Y 4096 1024 +Y 4096 2048 +Y 4096 4096 +Y 8192 128 +Y 8192 256 +Y 8192 512 +Y 8192 1024 +Y 8192 2048 +Y 8192 4096 +Y 16384 128 +Y 16384 256 +Y 16384 512 +Y 16384 1024 +Y 16384 2048 +Y 32768 128 +Y 32768 256 +Y 32768 512 +Y 32768 1024 +Y 65536 128 +Y 65536 256 +Y 131072 128 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/adaptive_points.jsonl b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/adaptive_points.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/adaptive_shapes.jsonl b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/adaptive_shapes.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/adaptive_summary.jsonl b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/adaptive_summary.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/adaptive_summary.md b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/adaptive_summary.md new file mode 100644 index 0000000..1ba5678 --- /dev/null +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/adaptive_summary.md @@ -0,0 +1,6 @@ +# Adaptive concurrency search summary + +| Engine | TP | DP | ISL | OSL | Stop | Saturation C | Best TPS C | Best Total TPS | Max successful C | +|---|---:|---:|---:|---:|---|---:|---:|---:|---:| + +`Saturation C` is the first point in the final low-gain streak. `Best TPS C` is the tested point with the highest observed Total TPS. diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/run_manifest.json b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/run_manifest.json new file mode 100644 index 0000000..03592b4 --- /dev/null +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/run_manifest.json @@ -0,0 +1,27 @@ +{ + "experiment": "dsv4_910c_vllm_tp_dp_matrix", + "engine": "vllm", + "run_id": "adaptive_20260729-112009", + "model": "/mnt/models/DeepSeek-V4-Flash-w8a8-mtp", + "hardware": "8x Ascend910 (16 dies, 64GB HBM/die)", + "matrix": "/mnt/yy/sskj/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/matrix.json", + "dataset": "random", + "tokenize_prompt": false, + "random_range_ratio": 1.0, + "search": { + "start_concurrency": 16, + "max_concurrency": 256, + "multiplier": 2, + "initial_backoff_concurrencies": [ + 8, + 1 + ], + "num_prompts_multiplier": 5, + "min_tps_gain_pct": 2.0, + "plateau_patience": 2, + "warmup_max_requests": 0, + "ttft_slo_ms": 4000.0, + "enable_ttft_slo_stop": 1, + "ttft_group_skip_ms": 8000.0 + } +} diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/shapes.tsv b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/shapes.tsv new file mode 100644 index 0000000..35b6705 --- /dev/null +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-112009/shapes.tsv @@ -0,0 +1,31 @@ +mark input_len output_len +Y 1024 128 +Y 1024 256 +Y 1024 512 +Y 1024 1024 +Y 1024 2048 +Y 1024 4096 +Y 4096 128 +Y 4096 256 +Y 4096 512 +Y 4096 1024 +Y 4096 2048 +Y 4096 4096 +Y 8192 128 +Y 8192 256 +Y 8192 512 +Y 8192 1024 +Y 8192 2048 +Y 8192 4096 +Y 16384 128 +Y 16384 256 +Y 16384 512 +Y 16384 1024 +Y 16384 2048 +Y 32768 128 +Y 32768 256 +Y 32768 512 +Y 32768 1024 +Y 65536 128 +Y 65536 256 +Y 131072 128 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/adaptive_points.jsonl b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/adaptive_points.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/adaptive_shapes.jsonl b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/adaptive_shapes.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/adaptive_summary.jsonl b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/adaptive_summary.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/adaptive_summary.md b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/adaptive_summary.md new file mode 100644 index 0000000..1ba5678 --- /dev/null +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/adaptive_summary.md @@ -0,0 +1,6 @@ +# Adaptive concurrency search summary + +| Engine | TP | DP | ISL | OSL | Stop | Saturation C | Best TPS C | Best Total TPS | Max successful C | +|---|---:|---:|---:|---:|---|---:|---:|---:|---:| + +`Saturation C` is the first point in the final low-gain streak. `Best TPS C` is the tested point with the highest observed Total TPS. diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/run_manifest.json b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/run_manifest.json new file mode 100644 index 0000000..3b630c9 --- /dev/null +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/run_manifest.json @@ -0,0 +1,27 @@ +{ + "experiment": "dsv4_910c_vllm_tp_dp_matrix", + "engine": "vllm", + "run_id": "adaptive_20260729-113439", + "model": "/mnt/models/DeepSeek-V4-Flash-w8a8-mtp", + "hardware": "8x Ascend910 (16 dies, 64GB HBM/die)", + "matrix": "/mnt/yy/sskj/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/matrix.json", + "dataset": "random", + "tokenize_prompt": false, + "random_range_ratio": 1.0, + "search": { + "start_concurrency": 16, + "max_concurrency": 256, + "multiplier": 2, + "initial_backoff_concurrencies": [ + 8, + 1 + ], + "num_prompts_multiplier": 5, + "min_tps_gain_pct": 2.0, + "plateau_patience": 2, + "warmup_max_requests": 0, + "ttft_slo_ms": 4000.0, + "enable_ttft_slo_stop": 1, + "ttft_group_skip_ms": 8000.0 + } +} diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/shapes.tsv b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/shapes.tsv new file mode 100644 index 0000000..35b6705 --- /dev/null +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113439/shapes.tsv @@ -0,0 +1,31 @@ +mark input_len output_len +Y 1024 128 +Y 1024 256 +Y 1024 512 +Y 1024 1024 +Y 1024 2048 +Y 1024 4096 +Y 4096 128 +Y 4096 256 +Y 4096 512 +Y 4096 1024 +Y 4096 2048 +Y 4096 4096 +Y 8192 128 +Y 8192 256 +Y 8192 512 +Y 8192 1024 +Y 8192 2048 +Y 8192 4096 +Y 16384 128 +Y 16384 256 +Y 16384 512 +Y 16384 1024 +Y 16384 2048 +Y 32768 128 +Y 32768 256 +Y 32768 512 +Y 32768 1024 +Y 65536 128 +Y 65536 256 +Y 131072 128 diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/adaptive_points.jsonl b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/adaptive_points.jsonl new file mode 100644 index 0000000..a4c867e --- /dev/null +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/adaptive_points.jsonl @@ -0,0 +1,3 @@ +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":38.5779427299276,"request_tps":2.073723851996349,"input_tps":2123.493224444261,"output_tps":265.43665305553264,"total_tps":2388.9298774997937,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1218.7625300139189,"ttft_p95_ms":3272.40676897927,"ttft_p99_ms":4362.865626341663,"tpot_p50_ms":49.4835413781574,"tpot_p95_ms":124.33025477457703,"tpot_p99_ms":79.74750237895067,"e2e_p50_ms":7354.705074976664,"e2e_p95_ms":9499.298503511818,"e2e_p99_ms":11670.241948680481,"itl_p50_ms":49.363330006599426,"itl_p95_ms":408.277097926474,"itl_p99_ms":448.8152776751667,"validation_errors":[],"timestamp":"2026-07-29T11:48:16+08:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/mnt/yy/sskj/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/tp4_dp2/raw_outputs/vllm_adaptive_c16_i1024_o128_a1.jsonl","detail_log":"/mnt/yy/sskj/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/tp4_dp2/logs/vllm_c16_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":48.83359965996351,"request_tps":3.2764326429775124,"input_tps":3355.0670264089727,"output_tps":419.3833783011216,"total_tps":3774.450404710094,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1306.9384199916385,"ttft_p95_ms":3467.1723649604246,"ttft_p99_ms":3896.632032970665,"tpot_p50_ms":61.83210448829428,"tpot_p95_ms":161.7796285074291,"tpot_p99_ms":98.65987394490887,"e2e_p50_ms":9634.722630027682,"e2e_p95_ms":11989.051085483516,"e2e_p99_ms":14206.873630409826,"itl_p50_ms":57.19536996912211,"itl_p95_ms":433.56416895403527,"itl_p99_ms":470.6485329172574,"validation_errors":[],"timestamp":"2026-07-29T11:49:30+08:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":57.997539,"plateau_streak":0,"error_type":"","raw_file":"/mnt/yy/sskj/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/tp4_dp2/raw_outputs/vllm_adaptive_c32_i1024_o128_a1.jsonl","detail_log":"/mnt/yy/sskj/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/tp4_dp2/logs/vllm_c32_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":54.53471646993421,"request_tps":4.4008663753173725,"input_tps":4506.487168324989,"output_tps":563.3108960406237,"total_tps":5069.798064365613,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1688.9932350022718,"ttft_p95_ms":4221.1321365553895,"ttft_p99_ms":5367.613992423278,"tpot_p50_ms":68.93910724380031,"tpot_p95_ms":164.9316360476405,"tpot_p99_ms":92.13227955046051,"e2e_p50_ms":11271.64134499617,"e2e_p95_ms":12833.146476407997,"e2e_p99_ms":13655.718730769586,"itl_p50_ms":61.67635991005227,"itl_p95_ms":453.4679424541537,"itl_p99_ms":482.7886850689538,"validation_errors":[],"timestamp":"2026-07-29T11:50:47+08:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":34.318842,"plateau_streak":0,"error_type":"","raw_file":"/mnt/yy/sskj/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/tp4_dp2/raw_outputs/vllm_adaptive_c48_i1024_o128_a1.jsonl","detail_log":"/mnt/yy/sskj/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/tp4_dp2/logs/vllm_c48_i1024_o128_a1.log"} diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/adaptive_shapes.jsonl b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/adaptive_shapes.jsonl new file mode 100644 index 0000000..051acef --- /dev/null +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/adaptive_shapes.jsonl @@ -0,0 +1 @@ +{"timestamp":"2026-07-29T11:50:47+08:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":256,"max_successful_concurrency":48,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":48,"best_total_tps":5069.798064365613,"last_total_tps":5069.798064365613} diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/run_manifest.json b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/run_manifest.json new file mode 100644 index 0000000..9a83ffa --- /dev/null +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/run_manifest.json @@ -0,0 +1,27 @@ +{ + "experiment": "dsv4_910c_vllm_tp_dp_matrix", + "engine": "vllm", + "run_id": "adaptive_20260729-113709", + "model": "/mnt/models/DeepSeek-V4-Flash-w8a8-mtp", + "hardware": "8x Ascend910 (16 dies, 64GB HBM/die)", + "matrix": "/mnt/yy/sskj/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/matrix.json", + "dataset": "random", + "tokenize_prompt": false, + "random_range_ratio": 1.0, + "search": { + "start_concurrency": 16, + "max_concurrency": 256, + "multiplier": 2, + "initial_backoff_concurrencies": [ + 8, + 1 + ], + "num_prompts_multiplier": 5, + "min_tps_gain_pct": 2.0, + "plateau_patience": 2, + "warmup_max_requests": 0, + "ttft_slo_ms": 4000.0, + "enable_ttft_slo_stop": 1, + "ttft_group_skip_ms": 8000.0 + } +} diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/shapes.tsv b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/shapes.tsv new file mode 100644 index 0000000..35b6705 --- /dev/null +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/adaptive_results/adaptive_20260729-113709/shapes.tsv @@ -0,0 +1,31 @@ +mark input_len output_len +Y 1024 128 +Y 1024 256 +Y 1024 512 +Y 1024 1024 +Y 1024 2048 +Y 1024 4096 +Y 4096 128 +Y 4096 256 +Y 4096 512 +Y 4096 1024 +Y 4096 2048 +Y 4096 4096 +Y 8192 128 +Y 8192 256 +Y 8192 512 +Y 8192 1024 +Y 8192 2048 +Y 8192 4096 +Y 16384 128 +Y 16384 256 +Y 16384 512 +Y 16384 1024 +Y 16384 2048 +Y 32768 128 +Y 32768 256 +Y 32768 512 +Y 32768 1024 +Y 65536 128 +Y 65536 256 +Y 131072 128 diff --git a/docs/DSV4_INFERENCE_COMPARISON_REPORT.md b/experiments/h200/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/DSV4_INFERENCE_COMPARISON_REPORT.md similarity index 100% rename from docs/DSV4_INFERENCE_COMPARISON_REPORT.md rename to experiments/h200/dsv4_h200_vllm_mtp_vs_default/results/20260708-160349/DSV4_INFERENCE_COMPARISON_REPORT.md diff --git a/experiments/h200/dsv4_h200_vllm_tp2_custom_bench/README.md b/experiments/h200/dsv4_h200_vllm_tp2_custom_bench/README.md index 20421ee..8ebec39 100644 --- a/experiments/h200/dsv4_h200_vllm_tp2_custom_bench/README.md +++ b/experiments/h200/dsv4_h200_vllm_tp2_custom_bench/README.md @@ -239,7 +239,7 @@ python3 bench_client.py --lb-strategy round_robin --ports 30005,30006,30007,3000 ### 结构化结果 (`results.json`) -遵循仓库统一的 [JSON Schema](../../BENCHMARK_WORKFLOW.md#final-json-schema)。 +遵循仓库统一的 [JSON Schema](../../../docs/BENCHMARK_WORKFLOW.md#final-json-schema)。 ### 人类可读报告 (`report.md`)