Add P800 sglang TP/DP matrix experiment
- New experiments/p800/dsv4_p800_sglang_tp_dp_matrix: TP8/DP1, TP4/DP2, TP2/DP4 matrix with smoke results; TP2/DP4 documents the weight-loading OOM root cause (274 GiB INT8 weights sharded only across TP group). - Launch args drop --ep-size/--chunked-prefill-size/--max-prefill-tokens/ --max-running-requests; experts fall back to TP sharding. - Move dsv4_p800_256k_4k_probe under experiments/p800/. - scripts/common: jq-free parsing fixes in adaptive_bench_lib.sh and parse_backend.py. - .gitignore: cover raw_outputs under nested platform experiment layout.
This commit is contained in:
yy-fighting
parent
8f89cf6dc2
commit
0fdd3749c7
2
.gitignore
vendored
2
.gitignore
vendored
@ -37,6 +37,8 @@ experiments/*/results/*/*/raw_outputs/
|
||||
experiments/*/results/*/*/logs/
|
||||
experiments/*/results/*/*.tsv
|
||||
experiments/*/results/*/*/*.tsv
|
||||
# 平台子目录布局（experiments/<platform>/<name>/results/...）下的 raw_outputs
|
||||
raw_outputs/
|
||||
|
||||
# 运行时目录（pid 文件、缓存、临时文件）
|
||||
experiments/*/runtime/
|
||||
|
||||
@ -16,14 +16,14 @@ without OOM or context-length errors.
|
||||
PLATFORM=kunlun_p800 \
|
||||
SKIP_MANAGE_SERVER=1 \
|
||||
PORT=30000 \
|
||||
bash experiments/dsv4_p800_256k_4k_probe/run_bench.sh
|
||||
bash experiments/p800/dsv4_p800_256k_4k_probe/run_bench.sh
|
||||
```
|
||||
|
||||
### Standalone (starts its own Docker container)
|
||||
|
||||
```bash
|
||||
PLATFORM=kunlun_p800 \
|
||||
bash experiments/dsv4_p800_256k_4k_probe/run_bench.sh
|
||||
bash experiments/p800/dsv4_p800_256k_4k_probe/run_bench.sh
|
||||
```
|
||||
|
||||
This will start a fresh `sglang-dsv4-flash` container with the baseline
|
||||
@ -34,7 +34,7 @@ To use the tuned config with EAGLE speculative decoding instead:
|
||||
|
||||
```bash
|
||||
SERVER_MODE=w8a8_int8 PLATFORM=kunlun_p800 \
|
||||
bash experiments/dsv4_p800_256k_4k_probe/run_bench.sh
|
||||
bash experiments/p800/dsv4_p800_256k_4k_probe/run_bench.sh
|
||||
```
|
||||
|
||||
## Configuration
|
||||
@ -46,7 +46,7 @@ Edit `config.env` or override via environment variables:
|
||||
PORT=30001 \
|
||||
MODEL_PATH=/data1/models/DeepSeek-V4-Flash-INT8 \
|
||||
PLATFORM=kunlun_p800 \
|
||||
bash experiments/dsv4_p800_256k_4k_probe/run_bench.sh
|
||||
bash experiments/p800/dsv4_p800_256k_4k_probe/run_bench.sh
|
||||
```
|
||||
|
||||
## Files
|
||||
@ -0,0 +1,31 @@
|
||||
{
|
||||
"metadata": {
|
||||
"experiment": "dsv4_p800_256k_4k_probe",
|
||||
"run_id": "20260708-144246",
|
||||
"timestamp": "2026-07-08T14:42:46+00:00",
|
||||
"model": "/data1/models/DeepSeek-V4-Flash-INT8",
|
||||
"backend": "sglang",
|
||||
"engine": "sglang-xpu",
|
||||
"hardware": "8x Kunlun P800 XPU",
|
||||
"accelerator": "Kunlun P800 XPU",
|
||||
"chip": "kunlun_p800",
|
||||
"script": "experiments/dsv4_p800_256k_4k_probe/run_bench.sh",
|
||||
"env": "",
|
||||
"git_commit": "89a999d",
|
||||
"git_dirty": "dirty",
|
||||
"description": "Kunlun P800 256k-input 4k-output OOM probe for DeepSeek-V4-Flash-INT8"
|
||||
},
|
||||
"config": {
|
||||
"tp": 8,
|
||||
"xpu_visible_devices": "0,1,2,3,4,5,6,7",
|
||||
"context_length": 270000,
|
||||
"server_mode": "w8a8_int8_baseline",
|
||||
"server_start_script": "experiments/dsv4_p800_256k_4k_probe/start_server.sh",
|
||||
"extra_launch_args": "--context-length 270000 --max-running-requests 4",
|
||||
"scenarios": [
|
||||
"2 262144 4096"
|
||||
],
|
||||
"num_prompts": 4
|
||||
},
|
||||
"scenarios": []
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
{
|
||||
"metadata": {
|
||||
"experiment": "dsv4_p800_256k_4k_probe",
|
||||
"run_id": "20260708-145540",
|
||||
"timestamp": "2026-07-08T14:55:40+00:00",
|
||||
"model": "/data1/models/DeepSeek-V4-Flash-INT8",
|
||||
"backend": "sglang",
|
||||
"engine": "sglang-xpu",
|
||||
"hardware": "8x Kunlun P800 XPU",
|
||||
"accelerator": "Kunlun P800 XPU",
|
||||
"chip": "kunlun_p800",
|
||||
"script": "experiments/dsv4_p800_256k_4k_probe/run_bench.sh",
|
||||
"env": "",
|
||||
"git_commit": "6ec62ff",
|
||||
"git_dirty": "dirty",
|
||||
"description": "Kunlun P800 256k-input 4k-output OOM probe for DeepSeek-V4-Flash-INT8"
|
||||
},
|
||||
"config": {
|
||||
"tp": 8,
|
||||
"xpu_visible_devices": "0,1,2,3,4,5,6,7",
|
||||
"context_length": 270000,
|
||||
"server_mode": "w8a8_int8_baseline",
|
||||
"server_start_script": "experiments/dsv4_p800_256k_4k_probe/start_server.sh",
|
||||
"extra_launch_args": "--context-length 270000 --max-running-requests 2 --mem-fraction-static 0.85",
|
||||
"scenarios": [
|
||||
"2 262144 4096"
|
||||
],
|
||||
"num_prompts": 4
|
||||
},
|
||||
"scenarios": []
|
||||
}
|
||||
@ -5,13 +5,13 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/server_docker.sh"
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/server_docker.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/bench_client_docker.sh"
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/bench_client_docker.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/config.env"
|
||||
|
||||
@ -5,7 +5,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/server_docker.sh"
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/server_docker.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/config.env"
|
||||
|
||||
74
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/README.md
Normal file
74
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/README.md
Normal file
@ -0,0 +1,74 @@
|
||||
# DSV4 P800 SGLang TP×DP Matrix
|
||||
|
||||
Kunlun P800 XPU + SGLang-XPU + `DeepSeek-V4-Flash-INT8` TP×DP matrix benchmark,
|
||||
modeled after `experiments/h20/dsv4_h20_sglang_tp_dp_matrix`.
|
||||
|
||||
## What it does
|
||||
|
||||
Runs each parallel configuration with its own server instance and sweeps the
|
||||
ISL/OSL matrix in `matrix.json`:
|
||||
|
||||
- TP=2, DP=4 → 2 XPUs per replica, 4 replicas
|
||||
- TP=4, DP=2 → 4 XPUs per replica, 2 replicas
|
||||
- TP=8, DP=1 → 8 XPUs, no data parallelism (production configuration)
|
||||
|
||||
Differences from the H20 version:
|
||||
|
||||
- Server runs in the P800 vendor image with the proven INT8 launch args
|
||||
(NSA attention backend, `w8a8_int8` quantization);
|
||||
the benchmark client runs inside the same container via `docker exec`.
|
||||
- ISL is capped at 131072 and concurrency ranges are lower, because P800
|
||||
INT8 sustains at most ~131k input tokens and far less concurrency.
|
||||
- XPU memory sampling uses `xpu-smi` instead of `nvidia-smi`.
|
||||
|
||||
`--dp-size` support on this image was probed empirically (2026-07-16,
|
||||
`results/smoke_tpdp`): TP=4/DP=2 starts and serves correctly (replicas land
|
||||
on disjoint XPU groups), while TP=2/DP=4 fails during model loading with a
|
||||
weight-loading OOM. Root cause: expert weights are sharded only across
|
||||
the TP group (this matrix passes no `--ep-size`, so experts fall back to
|
||||
TP sharding; the image also asserts `ep_size <= tp_size`, so EP can never
|
||||
exceed TP anyway) and DP replicas do not share expert shards. The INT8
|
||||
checkpoint holds ~264 GiB of routed-expert weights out of ~274 GiB total,
|
||||
so each rank carries roughly `274/TP` GiB:
|
||||
TP=8/DP=1 ≈ 34 GiB, TP=4/DP=2 ≈ 68 GiB, TP=2/DP=4 ≈ 137 GiB — more than
|
||||
the 96 GiB an XPU has, hence the OOM. TP=4 is therefore the
|
||||
minimum viable TP for this model on P800. Configs that cannot start are
|
||||
recorded as `SKIPPED_SERVICE_START_FAILED` and the run continues with the
|
||||
next config.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Full matrix (server per config -> warmup -> scenarios -> parse -> compare)
|
||||
bash experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh
|
||||
|
||||
# Dry run: print server args and scenario plan only
|
||||
DRY_RUN=1 bash experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh
|
||||
|
||||
# Adaptive concurrency search (find saturation point per shape)
|
||||
bash experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_adaptive_concurrency.sh
|
||||
```
|
||||
|
||||
Results land in `results/<RUN_ID>/` (fixed matrix) or
|
||||
`adaptive_results/<RUN_ID>/` (adaptive search).
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose |
|
||||
|---|---|
|
||||
| `config.env` | Experiment-level configuration (model, port, TP×DP configs, server settings) |
|
||||
| `matrix.json` | ISL/OSL matrix with Y/P/N marks and per-ISL concurrency ranges |
|
||||
| `generate_scenarios.py` | Expands matrix.json into a scenario TSV |
|
||||
| `start_sglang_docker.sh` | Start the P800 SGLang Docker container for a TP×DP config |
|
||||
| `start_sglang_dp.sh` | Alias for `start_sglang_docker.sh` (H20 naming parity) |
|
||||
| `run_bench.sh` | Orchestrator: server per config → warmup → scenarios → parse → compare |
|
||||
| `compare.py` | Cross-config comparison report (`comparison.md`) |
|
||||
| `adaptive_config.env` | Adaptive concurrency search settings |
|
||||
| `run_adaptive_concurrency.sh` | Adaptive saturation search (shared `adaptive_bench_lib.sh`) |
|
||||
| `run_adaptive_concurrency_add16.sh` | Same search starting at concurrency 16, step +16 |
|
||||
|
||||
## Platform
|
||||
|
||||
This experiment targets `platforms/kunlun_p800.env`. The container uses the
|
||||
dedicated name `sglang-dsv4-flash-tpdp` so it never interferes with the
|
||||
`sglang-dsv4-flash` container used by the other P800 experiments.
|
||||
@ -0,0 +1,57 @@
|
||||
# Adaptive concurrency search settings.
|
||||
#
|
||||
# For each fixed (TP, DP, ISL, OSL), probe:
|
||||
# C = start, start * multiplier, ... up to max
|
||||
# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for
|
||||
# PLATEAU_PATIENCE consecutive points.
|
||||
|
||||
SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
|
||||
# The server-side max-running-requests cap has been removed; 32 is kept as
|
||||
# the search upper bound to stay comparable with earlier runs.
|
||||
SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-32}"
|
||||
SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}"
|
||||
NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"
|
||||
|
||||
# A gain below 2% is treated as throughput saturation. Two consecutive
|
||||
# low-gain points prevent one noisy measurement from stopping the search.
|
||||
TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}"
|
||||
PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}"
|
||||
|
||||
# TTFT SLO early-stop settings.
|
||||
# When ttft_p95_ms exceeds TTFT_SLO_MS, stop searching the current (ISL, OSL)
|
||||
# shape and move on to the next scenario.
|
||||
TTFT_SLO_MS="${TTFT_SLO_MS:-4000}"
|
||||
ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}"
|
||||
|
||||
# Keep the same random workload semantics as the fixed matrix baseline.
|
||||
# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times
|
||||
# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to
|
||||
# random-ids to use generated token IDs without a ShareGPT seed dataset.
|
||||
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
|
||||
# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed
|
||||
# ISL/OSL points; lower values intentionally benchmark a length distribution.
|
||||
RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}"
|
||||
# NOTE: the P800 image's sglang.bench_serving does not support
|
||||
# --warmup-requests, so per-point warmup requests are not sent; the value is
|
||||
# still validated by adaptive_bench_lib.sh and kept for parity with H20.
|
||||
BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}"
|
||||
|
||||
# Reject a point if the completed request count or actual token lengths do not
|
||||
# match the requested workload.
|
||||
INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}"
|
||||
OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}"
|
||||
|
||||
MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}"
|
||||
SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}"
|
||||
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||||
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||||
|
||||
# Optional space-separated filters, useful for smoke tests:
|
||||
# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128"
|
||||
TP_LIST="${TP_LIST:-}"
|
||||
ISL_LIST="${ISL_LIST:-}"
|
||||
OSL_LIST="${OSL_LIST:-}"
|
||||
|
||||
DRY_RUN="${DRY_RUN:-0}"
|
||||
# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes.
|
||||
GRID_LIMIT="${GRID_LIMIT:-0}"
|
||||
@ -0,0 +1,22 @@
|
||||
{
|
||||
"experiment": "dsv4_p800_sglang_tp_dp_matrix",
|
||||
"engine": "sglang",
|
||||
"run_id": "adaptive_20260716-064715",
|
||||
"model": "/data1/models/DeepSeek-V4-Flash-INT8",
|
||||
"hardware": "8x Kunlun P800 XPU",
|
||||
"matrix": "/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/matrix.json",
|
||||
"dataset": "random",
|
||||
"tokenize_prompt": false,
|
||||
"random_range_ratio": 1.0,
|
||||
"search": {
|
||||
"start_concurrency": 16,
|
||||
"max_concurrency": 32,
|
||||
"multiplier": 2,
|
||||
"num_prompts_multiplier": 5,
|
||||
"min_tps_gain_pct": 2.0,
|
||||
"plateau_patience": 2,
|
||||
"warmup_max_requests": 0,
|
||||
"ttft_slo_ms": 4000.0,
|
||||
"enable_ttft_slo_stop": 1
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
mark input_len output_len
|
||||
Y 1024 128
|
||||
Y 1024 256
|
||||
Y 1024 512
|
||||
Y 1024 1024
|
||||
Y 1024 2048
|
||||
Y 1024 4096
|
||||
Y 4096 128
|
||||
Y 4096 256
|
||||
Y 4096 512
|
||||
Y 4096 1024
|
||||
Y 4096 2048
|
||||
Y 4096 4096
|
||||
Y 16384 128
|
||||
Y 16384 256
|
||||
Y 16384 512
|
||||
Y 16384 1024
|
||||
Y 16384 2048
|
||||
Y 65536 128
|
||||
Y 65536 256
|
||||
Y 65536 512
|
||||
Y 65536 1024
|
||||
Y 131072 128
|
||||
Y 131072 256
|
||||
Y 131072 512
|
||||
|
@ -0,0 +1 @@
|
||||
--host 0.0.0.0 --port 30014 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static 0.8 --tensor-parallel-size 2 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 140000 --dp-size 4
|
||||
@ -0,0 +1 @@
|
||||
--host 0.0.0.0 --port 30014 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static 0.8 --tensor-parallel-size 4 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 140000 --dp-size 2
|
||||
162
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/compare.py
Normal file
162
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/compare.py
Normal file
@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Cross TP×DP configuration comparison for dsv4_p800_sglang_tp_dp_matrix.
|
||||
|
||||
Usage:
|
||||
python3 compare.py --run-root results/<run_id> [--output comparison.md]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_result(result_root: Path) -> dict:
|
||||
path = result_root / "results.json"
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def slo_status(ttft_p95_ms: float, tpot_mean_ms: float,
|
||||
ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str:
|
||||
ttft_ok = ttft_p95_ms < ttft_limit_ms
|
||||
tpot_ok = tpot_mean_ms < tpot_limit_ms
|
||||
if ttft_ok and tpot_ok:
|
||||
return "PASS"
|
||||
if ttft_ok or tpot_ok:
|
||||
return "PARTIAL"
|
||||
return "FAIL"
|
||||
|
||||
|
||||
def gpu_memory_str(gpu: dict | None) -> str:
|
||||
if not gpu:
|
||||
return "-"
|
||||
peak = gpu.get("peak_used_mb", 0)
|
||||
total = gpu.get("memory_total_mb", 0)
|
||||
if total:
|
||||
return f"{peak:.0f}/{total:.0f} ({100*peak/total:.1f}%)"
|
||||
return f"{peak:.0f}"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--run-root", type=Path, required=True)
|
||||
parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md"))
|
||||
parser.add_argument("--ttft-limit", type=float, default=3000.0)
|
||||
parser.add_argument("--tpot-limit", type=float, default=50.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
# Discover configurations: tp*_dp* directories.
|
||||
configs = []
|
||||
for subdir in sorted(args.run_root.iterdir()):
|
||||
if not subdir.is_dir():
|
||||
continue
|
||||
name = subdir.name
|
||||
if not (name.startswith("tp") and "_dp" in name):
|
||||
continue
|
||||
results_json = subdir / "results.json"
|
||||
if not results_json.exists():
|
||||
continue
|
||||
configs.append((name, load_result(subdir)))
|
||||
|
||||
if not configs:
|
||||
print(f"No tp*_dp* results found under {args.run_root}")
|
||||
return
|
||||
|
||||
model = configs[0][1].get("metadata", {}).get("model", "unknown")
|
||||
hardware = configs[0][1].get("metadata", {}).get("hardware", "unknown")
|
||||
|
||||
# Group by scenario name.
|
||||
by_scenario: dict[str, dict[str, dict]] = defaultdict(dict)
|
||||
skipped: dict[str, dict[str, str]] = defaultdict(dict)
|
||||
for label, data in configs:
|
||||
for s in data.get("scenarios", []):
|
||||
key = s["name"]
|
||||
if s.get("status") == "skipped_oom" or "metrics" not in s:
|
||||
skipped[key][label] = s.get("note") or s.get("status", "skipped")
|
||||
else:
|
||||
by_scenario[key][label] = s
|
||||
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
f.write(f"# SGLang TP×DP matrix comparison ({hardware})\n\n")
|
||||
f.write("## Summary\n\n")
|
||||
f.write(f"- Model: `{model}`\n")
|
||||
f.write(f"- Hardware: {hardware}\n")
|
||||
f.write("- Backend: SGLang (Docker)\n")
|
||||
f.write("- Benchmark client: `sglang.bench_serving`\n")
|
||||
f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n")
|
||||
|
||||
# Configuration overview.
|
||||
f.write("### Configurations\n\n")
|
||||
f.write("| Config | TP | DP | XPUs/replica | Notes |\n")
|
||||
f.write("|---|---:|---:|---:|---|\n")
|
||||
for label, data in configs:
|
||||
cfg = data.get("config", {})
|
||||
tp = cfg.get("tp", "?")
|
||||
dp = cfg.get("dp", "?")
|
||||
f.write(f"| {label} | {tp} | {dp} | {tp} | server args recorded per ISL in results.json |\n")
|
||||
f.write("\n")
|
||||
|
||||
# Side-by-side table.
|
||||
f.write("## Side-by-side results\n\n")
|
||||
headers = [
|
||||
"Scenario", "ISL", "OSL", "Config", "Conc", "Req/s", "OutTok/s",
|
||||
"TTFT P95(ms)", "TTFT P99(ms)", "TPOT Mean(ms)", "TPOT P95(ms)",
|
||||
"TPOT P99(ms)", "E2E P99(ms)", "Peak GPU mem", "SLO"
|
||||
]
|
||||
f.write("| " + " | ".join(headers) + " |\n")
|
||||
f.write("|" + "|".join(["---"] * len(headers)) + "|\n")
|
||||
|
||||
for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, re.findall(r"\d+", x)))):
|
||||
_, isl, osl = re.findall(r"\d+", scenario_name)
|
||||
for label, data in configs:
|
||||
s = by_scenario[scenario_name].get(label)
|
||||
if s is None:
|
||||
if scenario_name in skipped and label in skipped[scenario_name]:
|
||||
note = skipped[scenario_name][label]
|
||||
f.write(f"| {scenario_name} | {isl} | {osl} | {label} | - | - | - | - | - | - | - | - | - | - | {note} |\n")
|
||||
continue
|
||||
cfg = s["config"]
|
||||
m = s["metrics"]
|
||||
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit)
|
||||
gpu = m.get("gpu_memory")
|
||||
f.write(
|
||||
f"| {scenario_name} | {isl} | {osl} | {label} | {cfg['concurrency']} | "
|
||||
f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
||||
f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
||||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
||||
f"{m['e2e_ms']['p99']:.2f} | {gpu_memory_str(gpu)} | {status} |\n"
|
||||
)
|
||||
|
||||
# Best throughput per ISL/OSL.
|
||||
f.write("\n## Best throughput per (ISL, OSL)\n\n")
|
||||
f.write("| ISL | OSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |\n")
|
||||
f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n")
|
||||
best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {}
|
||||
for scenario_name, backends in by_scenario.items():
|
||||
_, isl, osl = re.findall(r"\d+", scenario_name)
|
||||
isl_i, osl_i = int(isl), int(osl)
|
||||
for label, s in backends.items():
|
||||
m = s["metrics"]
|
||||
out_tok = m["output_token_throughput"]
|
||||
if (isl_i, osl_i) not in best_by_shape or out_tok > best_by_shape[(isl_i, osl_i)][0]:
|
||||
best_by_shape[(isl_i, osl_i)] = (out_tok, label, s)
|
||||
for (isl_i, osl_i), (out_tok, label, s) in sorted(best_by_shape.items()):
|
||||
m = s["metrics"]
|
||||
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit)
|
||||
f.write(
|
||||
f"| {isl_i} | {osl_i} | {label} | {s['config']['concurrency']} | "
|
||||
f"{out_tok:.2f} | {m['ttft_ms']['p95']:.2f} | {m['tpot_ms']['mean']:.2f} | {status} |\n"
|
||||
)
|
||||
|
||||
f.write("\n## Notes\n\n")
|
||||
f.write("- SLO check uses TTFT P95 and TPOT mean.\n")
|
||||
f.write("- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.\n")
|
||||
f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n")
|
||||
f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n")
|
||||
|
||||
print(f"Wrote comparison to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
85
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/config.env
Normal file
85
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/config.env
Normal file
@ -0,0 +1,85 @@
|
||||
# TP×DP matrix experiment for DeepSeek-V4-Flash-INT8 on Kunlun P800 (8 XPUs)
|
||||
# using SGLang-XPU. Tests SGLang with three parallel configurations:
|
||||
# TP=2, DP=4 -> 2 XPUs per replica, 4 replicas
|
||||
# TP=4, DP=2 -> 4 XPUs per replica, 2 replicas
|
||||
# TP=8, DP=1 -> 8 XPUs, no data parallelism
|
||||
|
||||
EXPERIMENT="dsv4_p800_sglang_tp_dp_matrix"
|
||||
MODEL_NAME="DeepSeek-V4-Flash-INT8"
|
||||
MODEL_PATH="${MODEL_PATH:-/data1/models/DeepSeek-V4-Flash-INT8}"
|
||||
SERVED_MODEL_NAME="deepseek-v4-flash-int8"
|
||||
|
||||
SGLANG_PORT="${SGLANG_PORT:-30014}"
|
||||
|
||||
# Dedicated container name so this experiment never touches the
|
||||
# `sglang-dsv4-flash` container used by the other P800 experiments.
|
||||
CONTAINER_NAME="sglang-dsv4-flash-tpdp"
|
||||
|
||||
# Python interpreter inside the P800 SGLang container (used to run the
|
||||
# benchmark client via `docker exec`).
|
||||
CONTAINER_PYTHON="${CONTAINER_PYTHON:-/root/miniconda/envs/python310_torch25_cuda/bin/python}"
|
||||
|
||||
export XPU_VISIBLE_DEVICES="${XPU_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||||
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||||
|
||||
# Runtime working directory for logs, pid files, and tmp. Defaults to a local
|
||||
# directory under this experiment so the benchmark is self-contained.
|
||||
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||
|
||||
# Parallel configurations to test. Format: "TP DP"
|
||||
# Note: TP=2/DP=4 fails at startup with a weight-loading OOM. Root cause:
|
||||
# expert weights are sharded only across the TP group (no --ep-size is
|
||||
# passed, so experts fall back to TP sharding), and DP replicas do not
|
||||
# share expert shards. The INT8 model holds ~264 GiB of routed-expert
|
||||
# weights, so TP=2 needs ~137 GiB per XPU (>96 GiB). TP=2/DP=4
|
||||
# is kept in the default list so the matrix records the failure and moves on.
|
||||
# Override with a space-separated list of comma pairs, e.g.
|
||||
# PARALLEL_CONFIGS_STR="8,1" to run only TP=8/DP=1.
|
||||
if [[ -n "${PARALLEL_CONFIGS_STR:-}" ]]; then
|
||||
declare -a PARALLEL_CONFIGS=()
|
||||
for pair in $PARALLEL_CONFIGS_STR; do
|
||||
PARALLEL_CONFIGS+=("${pair//,/ }")
|
||||
done
|
||||
else
|
||||
declare -a PARALLEL_CONFIGS=(
|
||||
"2 4"
|
||||
"4 2"
|
||||
"8 1"
|
||||
)
|
||||
fi
|
||||
|
||||
# SGLang server settings. P800 INT8 supports at most ~131k input tokens, so
|
||||
# the context length is capped well below the H20 1M setting.
|
||||
CONTEXT_LENGTH="${CONTEXT_LENGTH:-140000}"
|
||||
MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.8}"
|
||||
|
||||
DOCKER_IMAGE="${DOCKER_IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202}"
|
||||
|
||||
# Random dataset mounted into the container at /workspace/dummy_sharegpt.json.
|
||||
# DATASET_PATH is the host copy (used by the adaptive search for sanity
|
||||
# checks); the benchmark client always references CONTAINER_DATASET_PATH.
|
||||
PATCH_ROOT="${PATCH_ROOT:-/data1/yy/sskj/platforms/patches/kunlun_p800}"
|
||||
DATASET_PATH="${DATASET_PATH:-${PATCH_ROOT}/dummy_sharegpt.json}"
|
||||
CONTAINER_DATASET_PATH="${CONTAINER_DATASET_PATH:-/workspace/dummy_sharegpt.json}"
|
||||
|
||||
# Matrix and concurrency rules are defined in matrix.json by default.
|
||||
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
|
||||
MATRIX_MODE="${MATRIX_MODE:-Y}"
|
||||
|
||||
# Sampling density for concurrency (kept for parity with the H20 experiment;
|
||||
# generate_scenarios.py currently samples the low/high endpoints only).
|
||||
export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}"
|
||||
|
||||
# Per-scenario timeout to avoid hangs (seconds).
|
||||
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||||
|
||||
# XPU memory sampling interval (seconds).
|
||||
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||||
|
||||
# Dry-run mode: if 1, only log the server args and scenario plan without
|
||||
# starting any server or sending requests.
|
||||
DRY_RUN="${DRY_RUN:-0}"
|
||||
|
||||
# Per-config scenario limit for quick smoke tests. 0 = run all generated
|
||||
# scenarios.
|
||||
GRID_LIMIT="${GRID_LIMIT:-0}"
|
||||
@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate the scenario list for the TP×DP matrix experiment.
|
||||
|
||||
Reads matrix.json and prints TSV lines:
|
||||
|
||||
mark input_len output_len concurrency num_prompts
|
||||
|
||||
mark is one of Y/P/N. The caller (run_bench.sh) decides how to treat each.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def sample_concurrency(low: int, high: int, target: int) -> list[int]:
|
||||
"""Return only the low and high concurrency values in [low, high].
|
||||
|
||||
For the TP×DP matrix we only need the two endpoints of the concurrency
|
||||
range (e.g. 1 and 128 for ISL=1024). The `target` argument is kept for
|
||||
API compatibility but is ignored.
|
||||
"""
|
||||
assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}"
|
||||
if low == high:
|
||||
return [low]
|
||||
return [low, high]
|
||||
|
||||
|
||||
def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]:
|
||||
with open(matrix_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
matrix = data["matrix"]
|
||||
concurrency_cfg = data["concurrency"]
|
||||
|
||||
scenarios = []
|
||||
for isl_str in sorted(matrix.keys(), key=int):
|
||||
osl_map = matrix[isl_str]
|
||||
low = concurrency_cfg[isl_str]["low"]
|
||||
high = concurrency_cfg[isl_str]["high"]
|
||||
concurrencies = sample_concurrency(low, high, target_samples)
|
||||
|
||||
for osl_str in sorted(osl_map.keys(), key=int):
|
||||
mark = osl_map[osl_str]
|
||||
if mode == "Y" and mark != "Y":
|
||||
continue
|
||||
if mode == "Y+P" and mark not in ("Y", "P"):
|
||||
continue
|
||||
# mode == "all" keeps everything, including N.
|
||||
|
||||
for conc in concurrencies:
|
||||
scenarios.append(
|
||||
{
|
||||
"mark": mark,
|
||||
"input_len": int(isl_str),
|
||||
"output_len": int(osl_str),
|
||||
"concurrency": conc,
|
||||
"num_prompts": conc * 5,
|
||||
}
|
||||
)
|
||||
|
||||
return scenarios
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--matrix", type=Path, default=Path("matrix.json"))
|
||||
parser.add_argument("--mode", choices=["Y", "Y+P", "all"], default=None,
|
||||
help="Scenario selection mode. Defaults to matrix.mode.")
|
||||
parser.add_argument("--target-samples", type=int, default=0,
|
||||
help="Target number of concurrency samples. 0 = heuristic (6-8).")
|
||||
args = parser.parse_args()
|
||||
|
||||
with open(args.matrix, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
mode = args.mode if args.mode else data.get("mode", "Y+P")
|
||||
|
||||
target_samples = args.target_samples
|
||||
if target_samples <= 0:
|
||||
env_samples = os.getenv("CONCURRENCY_SAMPLES", "0")
|
||||
try:
|
||||
target_samples = int(env_samples)
|
||||
except ValueError:
|
||||
target_samples = 0
|
||||
if target_samples <= 0:
|
||||
target_samples = 7
|
||||
|
||||
scenarios = generate_scenarios(args.matrix, mode, target_samples)
|
||||
|
||||
print("mark\tinput_len\toutput_len\tconcurrency\tnum_prompts")
|
||||
for s in scenarios:
|
||||
print(f"{s['mark']}\t{s['input_len']}\t{s['output_len']}\t{s['concurrency']}\t{s['num_prompts']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
54
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/matrix.json
Normal file
54
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/matrix.json
Normal file
@ -0,0 +1,54 @@
|
||||
{
|
||||
"comment": "ISL/OSL matrix for dsv4_p800_sglang_tp_dp_matrix. Y=must test, P=optional (record skipped on failure), N=skip.",
|
||||
"mode": "Y",
|
||||
"notes": "P800 INT8 sustains at most ~131k input tokens, so ISL stops at 131072 and concurrency ranges are much lower than the H20 matrix.",
|
||||
"matrix": {
|
||||
"1024": {
|
||||
"128": "Y",
|
||||
"256": "Y",
|
||||
"512": "Y",
|
||||
"1024": "Y",
|
||||
"2048": "Y",
|
||||
"4096": "Y"
|
||||
},
|
||||
"4096": {
|
||||
"128": "Y",
|
||||
"256": "Y",
|
||||
"512": "Y",
|
||||
"1024": "Y",
|
||||
"2048": "Y",
|
||||
"4096": "Y"
|
||||
},
|
||||
"16384": {
|
||||
"128": "Y",
|
||||
"256": "Y",
|
||||
"512": "Y",
|
||||
"1024": "Y",
|
||||
"2048": "Y",
|
||||
"4096": "P"
|
||||
},
|
||||
"65536": {
|
||||
"128": "Y",
|
||||
"256": "Y",
|
||||
"512": "Y",
|
||||
"1024": "Y",
|
||||
"2048": "P",
|
||||
"4096": "N"
|
||||
},
|
||||
"131072": {
|
||||
"128": "Y",
|
||||
"256": "Y",
|
||||
"512": "Y",
|
||||
"1024": "P",
|
||||
"2048": "N",
|
||||
"4096": "N"
|
||||
}
|
||||
},
|
||||
"concurrency": {
|
||||
"1024": { "low": 1, "high": 32 },
|
||||
"4096": { "low": 1, "high": 16 },
|
||||
"16384": { "low": 1, "high": 8 },
|
||||
"65536": { "low": 1, "high": 4 },
|
||||
"131072": { "low": 1, "high": 2 }
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,34 @@
|
||||
# SGLang TP×DP matrix comparison (8x Kunlun P800 XPU)
|
||||
|
||||
## Summary
|
||||
|
||||
- Model: `/data1/models/DeepSeek-V4-Flash-INT8`
|
||||
- Hardware: 8x Kunlun P800 XPU
|
||||
- Backend: SGLang (Docker)
|
||||
- Benchmark client: `sglang.bench_serving`
|
||||
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
|
||||
|
||||
### Configurations
|
||||
|
||||
| Config | TP | DP | XPUs/replica | Notes |
|
||||
|---|---:|---:|---:|---|
|
||||
| tp8_dp1 | 8 | 1 | 8 | server args recorded per ISL in results.json |
|
||||
|
||||
## Side-by-side results
|
||||
|
||||
| Scenario | ISL | OSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| c1_i1024_o128 | 1024 | 128 | tp8_dp1 | 1 | 0.28 | 35.50 | 0.00 | 251.79 | 26.41 | 0.00 | 26.67 | 3637.43 | 92332/98304 (93.9%) | PASS |
|
||||
|
||||
## Best throughput per (ISL, OSL)
|
||||
|
||||
| ISL | OSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |
|
||||
|---:|---:|---|---:|---:|---:|---:|---:|
|
||||
| 1024 | 128 | tp8_dp1 | 1 | 35.50 | 0.00 | 26.41 | PASS |
|
||||
|
||||
## Notes
|
||||
|
||||
- SLO check uses TTFT P95 and TPOT mean.
|
||||
- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.
|
||||
- `Peak GPU mem` shows peak used / total MB and utilization percentage.
|
||||
- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.
|
||||
@ -0,0 +1,67 @@
|
||||
# 8x Kunlun P800 XPU SGLANG Benchmark Report
|
||||
|
||||
- Result root: `results/smoke_tp8/tp8_dp1`
|
||||
- Model: `/data1/models/DeepSeek-V4-Flash-INT8`
|
||||
- Backend: SGLANG
|
||||
- Benchmark client: `sglang.bench_serving --backend sglang`
|
||||
|
||||
## Results
|
||||
|
||||
| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Peak GPU mem | SLO |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| c1_i1024_o128 | main | 1 | 1024 | 128 | 18.03 | 5 | 0.28 | 284.02 | 35.50 | 319.52 | 249.58 | 0.00 | 251.79 | 26.41 | 0.00 | 26.67 | 3603.09 | 0.00 | 3637.43 | 92332/98304 MiB (93.9%) | ✅ |
|
||||
|
||||
SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.
|
||||
|
||||
## Skipped or failed scenarios
|
||||
|
||||
| Scenario | Status | Note |
|
||||
|---|---|---|
|
||||
| c32_i1024_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i1024_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c32_i1024_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i1024_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c32_i1024_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i1024_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c32_i1024_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i1024_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c32_i1024_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i1024_o4096 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c32_i1024_o4096 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i4096_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c16_i4096_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i4096_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c16_i4096_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i4096_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c16_i4096_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i4096_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c16_i4096_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i4096_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c16_i4096_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i4096_o4096 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c16_i4096_o4096 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i16384_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c8_i16384_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i16384_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c8_i16384_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i16384_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c8_i16384_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i16384_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c8_i16384_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i16384_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c8_i16384_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i65536_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c4_i65536_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i65536_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c4_i65536_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i65536_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c4_i65536_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i65536_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c4_i65536_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i131072_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c2_i131072_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i131072_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c2_i131072_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i131072_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c2_i131072_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
|
||||
@ -0,0 +1,702 @@
|
||||
{
|
||||
"metadata": {
|
||||
"experiment": "dsv4_p800_sglang_tp_dp_matrix_tp8_dp1",
|
||||
"run_id": "smoke_tp8",
|
||||
"timestamp": "2026-07-16T05:45:26+00:00",
|
||||
"model": "/data1/models/DeepSeek-V4-Flash-INT8",
|
||||
"backend": "sglang",
|
||||
"engine": "sglang-xpu",
|
||||
"hardware": "8x Kunlun P800 XPU",
|
||||
"accelerator": "Kunlun P800 XPU",
|
||||
"chip": "kunlun_p800",
|
||||
"script": "experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh",
|
||||
"env": "iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202",
|
||||
"git_commit": "f4f4093",
|
||||
"git_dirty": "dirty",
|
||||
"description": "P800 SGLang TP×DP matrix for DeepSeek-V4-Flash-INT8"
|
||||
},
|
||||
"config": {
|
||||
"tp": 8,
|
||||
"dp": 1,
|
||||
"xpu_visible_devices": "0,1,2,3,4,5,6,7",
|
||||
"backend": "sglang",
|
||||
"engine": "sglang-xpu",
|
||||
"server_start_script": "experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_docker.sh",
|
||||
"server_args": "--host 0.0.0.0 --port 30014 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 140000 --max-running-requests 32 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 140000"
|
||||
},
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "c1_i1024_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"metrics": {
|
||||
"success": 5,
|
||||
"failed": 0,
|
||||
"duration_s": 18.0267706339946,
|
||||
"request_throughput": 0.2773652642238138,
|
||||
"input_token_throughput": 284.02203056518533,
|
||||
"output_token_throughput": 35.50275382064817,
|
||||
"total_token_throughput": 319.52478438583347,
|
||||
"total_input_tokens": 5120,
|
||||
"total_output_tokens": 640,
|
||||
"e2e_ms": {
|
||||
"mean": 3603.0893054325134,
|
||||
"p50": 3616.7099910089746,
|
||||
"p90": 3632.4909622315317,
|
||||
"p95": 0.0,
|
||||
"p99": 3637.4272649595514
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 249.58492522127926,
|
||||
"p50": 250.24159904569387,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 251.78673897404224
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 26.405546300875862,
|
||||
"p50": 26.52918022881284,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 26.670257044684757
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 26.405476773429047,
|
||||
"p50": 24.089168990030885,
|
||||
"p90": 0.0,
|
||||
"p95": 47.44513131445274,
|
||||
"p99": 53.76050190534443
|
||||
},
|
||||
"gpu_memory": {
|
||||
"peak_used_mb": 92332.0,
|
||||
"avg_used_mb": 92332.0,
|
||||
"peak_utilization_pct": 100.0,
|
||||
"memory_total_mb": 98304.0
|
||||
}
|
||||
},
|
||||
"slo_status": {
|
||||
"ttft_p95_ok": true,
|
||||
"tpot_mean_ok": true,
|
||||
"overall": "✅"
|
||||
},
|
||||
"raw_file": "results/smoke_tp8/tp8_dp1/raw_outputs/sglang_main_1_1024_128.jsonl"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i1024_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i1024_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i1024_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i1024_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i1024_o4096",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 4096,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o4096",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 4096,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o4096",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 4096,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o4096",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 4096,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i16384_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 16384,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c8_i16384_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 8,
|
||||
"input_len": 16384,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 40
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i16384_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 16384,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c8_i16384_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 8,
|
||||
"input_len": 16384,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 40
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i16384_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 16384,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c8_i16384_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 8,
|
||||
"input_len": 16384,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 40
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i16384_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 16384,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c8_i16384_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 8,
|
||||
"input_len": 16384,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 40
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i16384_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 16384,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c8_i16384_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 8,
|
||||
"input_len": 16384,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 40
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i65536_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 65536,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c4_i65536_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 4,
|
||||
"input_len": 65536,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 20
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i65536_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 65536,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c4_i65536_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 4,
|
||||
"input_len": 65536,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 20
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i65536_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 65536,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c4_i65536_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 4,
|
||||
"input_len": 65536,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 20
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i65536_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 65536,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c4_i65536_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 4,
|
||||
"input_len": 65536,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 20
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i131072_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 131072,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c2_i131072_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 2,
|
||||
"input_len": 131072,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 10
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i131072_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 131072,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c2_i131072_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 2,
|
||||
"input_len": 131072,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 10
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i131072_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 131072,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c2_i131072_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 2,
|
||||
"input_len": 131072,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 10
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,49 @@
|
||||
mark input_len output_len concurrency num_prompts
|
||||
Y 1024 128 1 5
|
||||
Y 1024 128 32 160
|
||||
Y 1024 256 1 5
|
||||
Y 1024 256 32 160
|
||||
Y 1024 512 1 5
|
||||
Y 1024 512 32 160
|
||||
Y 1024 1024 1 5
|
||||
Y 1024 1024 32 160
|
||||
Y 1024 2048 1 5
|
||||
Y 1024 2048 32 160
|
||||
Y 1024 4096 1 5
|
||||
Y 1024 4096 32 160
|
||||
Y 4096 128 1 5
|
||||
Y 4096 128 16 80
|
||||
Y 4096 256 1 5
|
||||
Y 4096 256 16 80
|
||||
Y 4096 512 1 5
|
||||
Y 4096 512 16 80
|
||||
Y 4096 1024 1 5
|
||||
Y 4096 1024 16 80
|
||||
Y 4096 2048 1 5
|
||||
Y 4096 2048 16 80
|
||||
Y 4096 4096 1 5
|
||||
Y 4096 4096 16 80
|
||||
Y 16384 128 1 5
|
||||
Y 16384 128 8 40
|
||||
Y 16384 256 1 5
|
||||
Y 16384 256 8 40
|
||||
Y 16384 512 1 5
|
||||
Y 16384 512 8 40
|
||||
Y 16384 1024 1 5
|
||||
Y 16384 1024 8 40
|
||||
Y 16384 2048 1 5
|
||||
Y 16384 2048 8 40
|
||||
Y 65536 128 1 5
|
||||
Y 65536 128 4 20
|
||||
Y 65536 256 1 5
|
||||
Y 65536 256 4 20
|
||||
Y 65536 512 1 5
|
||||
Y 65536 512 4 20
|
||||
Y 65536 1024 1 5
|
||||
Y 65536 1024 4 20
|
||||
Y 131072 128 1 5
|
||||
Y 131072 128 2 10
|
||||
Y 131072 256 1 5
|
||||
Y 131072 256 2 10
|
||||
Y 131072 512 1 5
|
||||
Y 131072 512 2 10
|
||||
|
@ -0,0 +1,36 @@
|
||||
# SGLang TP×DP matrix comparison (8x Kunlun P800 XPU)
|
||||
|
||||
## Summary
|
||||
|
||||
- Model: `/data1/models/DeepSeek-V4-Flash-INT8`
|
||||
- Hardware: 8x Kunlun P800 XPU
|
||||
- Backend: SGLang (Docker)
|
||||
- Benchmark client: `sglang.bench_serving`
|
||||
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
|
||||
|
||||
### Configurations
|
||||
|
||||
| Config | TP | DP | XPUs/replica | Notes |
|
||||
|---|---:|---:|---:|---|
|
||||
| tp2_dp4 | 2 | 4 | 2 | server args recorded per ISL in results.json |
|
||||
| tp4_dp2 | 4 | 2 | 4 | server args recorded per ISL in results.json |
|
||||
|
||||
## Side-by-side results
|
||||
|
||||
| Scenario | ISL | OSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| c1_i1024_o128 | 1024 | 128 | tp2_dp4 | - | - | - | - | - | - | - | - | - | - | service failed to start |
|
||||
| c1_i1024_o128 | 1024 | 128 | tp4_dp2 | 1 | 0.23 | 29.80 | 0.00 | 305.76 | 31.44 | 0.00 | 31.55 | 4305.40 | 82314/98304 (83.7%) | PASS |
|
||||
|
||||
## Best throughput per (ISL, OSL)
|
||||
|
||||
| ISL | OSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |
|
||||
|---:|---:|---|---:|---:|---:|---:|---:|
|
||||
| 1024 | 128 | tp4_dp2 | 1 | 29.80 | 0.00 | 31.44 | PASS |
|
||||
|
||||
## Notes
|
||||
|
||||
- SLO check uses TTFT P95 and TPOT mean.
|
||||
- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.
|
||||
- `Peak GPU mem` shows peak used / total MB and utilization percentage.
|
||||
- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.
|
||||
@ -0,0 +1,653 @@
|
||||
{
|
||||
"metadata": {
|
||||
"experiment": "dsv4_p800_sglang_tp_dp_matrix_tp2_dp4",
|
||||
"run_id": "smoke_tpdp",
|
||||
"timestamp": "2026-07-16T06:00:00+00:00",
|
||||
"model": "/data1/models/DeepSeek-V4-Flash-INT8",
|
||||
"backend": "sglang",
|
||||
"engine": "sglang-xpu",
|
||||
"hardware": "8x Kunlun P800 XPU",
|
||||
"accelerator": "Kunlun P800 XPU",
|
||||
"chip": "kunlun_p800",
|
||||
"script": "experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh",
|
||||
"env": "iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202",
|
||||
"git_commit": "f4f4093",
|
||||
"git_dirty": "dirty",
|
||||
"description": "P800 SGLang TP×DP matrix for DeepSeek-V4-Flash-INT8"
|
||||
},
|
||||
"config": {
|
||||
"tp": 2,
|
||||
"dp": 4,
|
||||
"xpu_visible_devices": "0,1,2,3,4,5,6,7",
|
||||
"backend": "sglang",
|
||||
"engine": "sglang-xpu",
|
||||
"server_start_script": "experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_docker.sh",
|
||||
"server_args": "--host 0.0.0.0 --port 30014 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 140000 --max-running-requests 32 --tensor-parallel-size 2 --ep-size 2 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 140000 --dp-size 4"
|
||||
},
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "c1_i1024_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i1024_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i1024_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i1024_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i1024_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i1024_o4096",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 4096,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o4096",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 4096,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o4096",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 4096,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o4096",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 4096,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i16384_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 16384,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c8_i16384_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 8,
|
||||
"input_len": 16384,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 40
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i16384_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 16384,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c8_i16384_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 8,
|
||||
"input_len": 16384,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 40
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i16384_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 16384,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c8_i16384_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 8,
|
||||
"input_len": 16384,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 40
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i16384_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 16384,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c8_i16384_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 8,
|
||||
"input_len": 16384,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 40
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i16384_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 16384,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c8_i16384_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 8,
|
||||
"input_len": 16384,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 40
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i65536_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 65536,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c4_i65536_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 4,
|
||||
"input_len": 65536,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 20
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i65536_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 65536,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c4_i65536_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 4,
|
||||
"input_len": 65536,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 20
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i65536_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 65536,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c4_i65536_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 4,
|
||||
"input_len": 65536,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 20
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i65536_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 65536,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c4_i65536_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 4,
|
||||
"input_len": 65536,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 20
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i131072_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 131072,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c2_i131072_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 2,
|
||||
"input_len": 131072,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 10
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i131072_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 131072,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c2_i131072_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 2,
|
||||
"input_len": 131072,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 10
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c1_i131072_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 131072,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
},
|
||||
{
|
||||
"name": "c2_i131072_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 2,
|
||||
"input_len": 131072,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 10
|
||||
},
|
||||
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||
"note": "service failed to start"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,49 @@
|
||||
mark input_len output_len concurrency num_prompts
|
||||
Y 1024 128 1 5
|
||||
Y 1024 128 32 160
|
||||
Y 1024 256 1 5
|
||||
Y 1024 256 32 160
|
||||
Y 1024 512 1 5
|
||||
Y 1024 512 32 160
|
||||
Y 1024 1024 1 5
|
||||
Y 1024 1024 32 160
|
||||
Y 1024 2048 1 5
|
||||
Y 1024 2048 32 160
|
||||
Y 1024 4096 1 5
|
||||
Y 1024 4096 32 160
|
||||
Y 4096 128 1 5
|
||||
Y 4096 128 16 80
|
||||
Y 4096 256 1 5
|
||||
Y 4096 256 16 80
|
||||
Y 4096 512 1 5
|
||||
Y 4096 512 16 80
|
||||
Y 4096 1024 1 5
|
||||
Y 4096 1024 16 80
|
||||
Y 4096 2048 1 5
|
||||
Y 4096 2048 16 80
|
||||
Y 4096 4096 1 5
|
||||
Y 4096 4096 16 80
|
||||
Y 16384 128 1 5
|
||||
Y 16384 128 8 40
|
||||
Y 16384 256 1 5
|
||||
Y 16384 256 8 40
|
||||
Y 16384 512 1 5
|
||||
Y 16384 512 8 40
|
||||
Y 16384 1024 1 5
|
||||
Y 16384 1024 8 40
|
||||
Y 16384 2048 1 5
|
||||
Y 16384 2048 8 40
|
||||
Y 65536 128 1 5
|
||||
Y 65536 128 4 20
|
||||
Y 65536 256 1 5
|
||||
Y 65536 256 4 20
|
||||
Y 65536 512 1 5
|
||||
Y 65536 512 4 20
|
||||
Y 65536 1024 1 5
|
||||
Y 65536 1024 4 20
|
||||
Y 131072 128 1 5
|
||||
Y 131072 128 2 10
|
||||
Y 131072 256 1 5
|
||||
Y 131072 256 2 10
|
||||
Y 131072 512 1 5
|
||||
Y 131072 512 2 10
|
||||
|
@ -0,0 +1,67 @@
|
||||
# 8x Kunlun P800 XPU SGLANG Benchmark Report
|
||||
|
||||
- Result root: `/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp4_dp2`
|
||||
- Model: `/data1/models/DeepSeek-V4-Flash-INT8`
|
||||
- Backend: SGLANG
|
||||
- Benchmark client: `sglang.bench_serving --backend sglang`
|
||||
|
||||
## Results
|
||||
|
||||
| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Peak GPU mem | SLO |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| c1_i1024_o128 | main | 1 | 1024 | 128 | 21.48 | 5 | 0.23 | 238.40 | 29.80 | 268.20 | 299.56 | 0.00 | 305.76 | 31.44 | 0.00 | 31.55 | 4292.17 | 0.00 | 4305.40 | 82314/98304 MiB (83.7%) | ✅ |
|
||||
|
||||
SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.
|
||||
|
||||
## Skipped or failed scenarios
|
||||
|
||||
| Scenario | Status | Note |
|
||||
|---|---|---|
|
||||
| c32_i1024_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i1024_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c32_i1024_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i1024_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c32_i1024_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i1024_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c32_i1024_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i1024_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c32_i1024_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i1024_o4096 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c32_i1024_o4096 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i4096_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c16_i4096_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i4096_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c16_i4096_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i4096_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c16_i4096_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i4096_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c16_i4096_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i4096_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c16_i4096_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i4096_o4096 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c16_i4096_o4096 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i16384_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c8_i16384_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i16384_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c8_i16384_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i16384_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c8_i16384_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i16384_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c8_i16384_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i16384_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c8_i16384_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i65536_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c4_i65536_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i65536_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c4_i65536_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i65536_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c4_i65536_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i65536_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c4_i65536_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i131072_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c2_i131072_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i131072_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c2_i131072_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c1_i131072_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
| c2_i131072_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached |
|
||||
|
||||
@ -0,0 +1,702 @@
|
||||
{
|
||||
"metadata": {
|
||||
"experiment": "dsv4_p800_sglang_tp_dp_matrix_tp4_dp2",
|
||||
"run_id": "smoke_tpdp",
|
||||
"timestamp": "2026-07-16T06:00:58+00:00",
|
||||
"model": "/data1/models/DeepSeek-V4-Flash-INT8",
|
||||
"backend": "sglang",
|
||||
"engine": "sglang-xpu",
|
||||
"hardware": "8x Kunlun P800 XPU",
|
||||
"accelerator": "Kunlun P800 XPU",
|
||||
"chip": "kunlun_p800",
|
||||
"script": "experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh",
|
||||
"env": "iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202",
|
||||
"git_commit": "f4f4093",
|
||||
"git_dirty": "dirty",
|
||||
"description": "P800 SGLang TP×DP matrix for DeepSeek-V4-Flash-INT8"
|
||||
},
|
||||
"config": {
|
||||
"tp": 4,
|
||||
"dp": 2,
|
||||
"xpu_visible_devices": "0,1,2,3,4,5,6,7",
|
||||
"backend": "sglang",
|
||||
"engine": "sglang-xpu",
|
||||
"server_start_script": "experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_docker.sh",
|
||||
"server_args": "--host 0.0.0.0 --port 30014 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 140000 --max-running-requests 32 --tensor-parallel-size 4 --ep-size 4 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 140000 --dp-size 2"
|
||||
},
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "c1_i1024_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"metrics": {
|
||||
"success": 5,
|
||||
"failed": 0,
|
||||
"duration_s": 21.476578783942387,
|
||||
"request_throughput": 0.23281175508914861,
|
||||
"input_token_throughput": 238.39923721128818,
|
||||
"output_token_throughput": 29.799904651411023,
|
||||
"total_token_throughput": 268.1991418626992,
|
||||
"total_input_tokens": 5120,
|
||||
"total_output_tokens": 640,
|
||||
"e2e_ms": {
|
||||
"mean": 4292.167780222371,
|
||||
"p50": 4291.483911918476,
|
||||
"p90": 4302.627834840678,
|
||||
"p95": 0.0,
|
||||
"p99": 4305.40240991395
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": 299.5579913724214,
|
||||
"p50": 298.9764349767938,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 305.76277466956526
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": 31.43787235314921,
|
||||
"p50": 31.393611976898857,
|
||||
"p90": 0.0,
|
||||
"p95": 0.0,
|
||||
"p99": 31.546158453038945
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": 31.43780216387671,
|
||||
"p50": 29.1481880703941,
|
||||
"p90": 0.0,
|
||||
"p95": 55.60536030679939,
|
||||
"p99": 62.21151448087765
|
||||
},
|
||||
"gpu_memory": {
|
||||
"peak_used_mb": 82314.0,
|
||||
"avg_used_mb": 82118.0,
|
||||
"peak_utilization_pct": 100.0,
|
||||
"memory_total_mb": 98304.0
|
||||
}
|
||||
},
|
||||
"slo_status": {
|
||||
"ttft_p95_ok": true,
|
||||
"tpot_mean_ok": true,
|
||||
"overall": "✅"
|
||||
},
|
||||
"raw_file": "/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp4_dp2/raw_outputs/sglang_main_1_1024_128.jsonl"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i1024_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i1024_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i1024_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i1024_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i1024_o4096",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 1024,
|
||||
"output_len": 4096,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c32_i1024_o4096",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 32,
|
||||
"input_len": 1024,
|
||||
"output_len": 4096,
|
||||
"dataset": "random",
|
||||
"num_prompts": 160
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i4096_o4096",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 4096,
|
||||
"output_len": 4096,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c16_i4096_o4096",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 16,
|
||||
"input_len": 4096,
|
||||
"output_len": 4096,
|
||||
"dataset": "random",
|
||||
"num_prompts": 80
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i16384_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 16384,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c8_i16384_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 8,
|
||||
"input_len": 16384,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 40
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i16384_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 16384,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c8_i16384_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 8,
|
||||
"input_len": 16384,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 40
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i16384_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 16384,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c8_i16384_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 8,
|
||||
"input_len": 16384,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 40
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i16384_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 16384,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c8_i16384_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 8,
|
||||
"input_len": 16384,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 40
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i16384_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 16384,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c8_i16384_o2048",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 8,
|
||||
"input_len": 16384,
|
||||
"output_len": 2048,
|
||||
"dataset": "random",
|
||||
"num_prompts": 40
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i65536_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 65536,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c4_i65536_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 4,
|
||||
"input_len": 65536,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 20
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i65536_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 65536,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c4_i65536_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 4,
|
||||
"input_len": 65536,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 20
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i65536_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 65536,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c4_i65536_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 4,
|
||||
"input_len": 65536,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 20
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i65536_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 65536,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c4_i65536_o1024",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 4,
|
||||
"input_len": 65536,
|
||||
"output_len": 1024,
|
||||
"dataset": "random",
|
||||
"num_prompts": 20
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i131072_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 131072,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c2_i131072_o128",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 2,
|
||||
"input_len": 131072,
|
||||
"output_len": 128,
|
||||
"dataset": "random",
|
||||
"num_prompts": 10
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i131072_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 131072,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c2_i131072_o256",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 2,
|
||||
"input_len": 131072,
|
||||
"output_len": 256,
|
||||
"dataset": "random",
|
||||
"num_prompts": 10
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c1_i131072_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 1,
|
||||
"input_len": 131072,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 5
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
},
|
||||
{
|
||||
"name": "c2_i131072_o512",
|
||||
"config": {
|
||||
"phase": "main",
|
||||
"concurrency": 2,
|
||||
"input_len": 131072,
|
||||
"output_len": 512,
|
||||
"dataset": "random",
|
||||
"num_prompts": 10
|
||||
},
|
||||
"status": "SKIPPED_GRID_LIMIT",
|
||||
"note": "GRID_LIMIT reached"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,49 @@
|
||||
mark input_len output_len concurrency num_prompts
|
||||
Y 1024 128 1 5
|
||||
Y 1024 128 32 160
|
||||
Y 1024 256 1 5
|
||||
Y 1024 256 32 160
|
||||
Y 1024 512 1 5
|
||||
Y 1024 512 32 160
|
||||
Y 1024 1024 1 5
|
||||
Y 1024 1024 32 160
|
||||
Y 1024 2048 1 5
|
||||
Y 1024 2048 32 160
|
||||
Y 1024 4096 1 5
|
||||
Y 1024 4096 32 160
|
||||
Y 4096 128 1 5
|
||||
Y 4096 128 16 80
|
||||
Y 4096 256 1 5
|
||||
Y 4096 256 16 80
|
||||
Y 4096 512 1 5
|
||||
Y 4096 512 16 80
|
||||
Y 4096 1024 1 5
|
||||
Y 4096 1024 16 80
|
||||
Y 4096 2048 1 5
|
||||
Y 4096 2048 16 80
|
||||
Y 4096 4096 1 5
|
||||
Y 4096 4096 16 80
|
||||
Y 16384 128 1 5
|
||||
Y 16384 128 8 40
|
||||
Y 16384 256 1 5
|
||||
Y 16384 256 8 40
|
||||
Y 16384 512 1 5
|
||||
Y 16384 512 8 40
|
||||
Y 16384 1024 1 5
|
||||
Y 16384 1024 8 40
|
||||
Y 16384 2048 1 5
|
||||
Y 16384 2048 8 40
|
||||
Y 65536 128 1 5
|
||||
Y 65536 128 4 20
|
||||
Y 65536 256 1 5
|
||||
Y 65536 256 4 20
|
||||
Y 65536 512 1 5
|
||||
Y 65536 512 4 20
|
||||
Y 65536 1024 1 5
|
||||
Y 65536 1024 4 20
|
||||
Y 131072 128 1 5
|
||||
Y 131072 128 2 10
|
||||
Y 131072 256 1 5
|
||||
Y 131072 256 2 10
|
||||
Y 131072 512 1 5
|
||||
Y 131072 512 2 10
|
||||
|
149
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_adaptive_concurrency.sh
Executable file
149
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_adaptive_concurrency.sh
Executable file
@ -0,0 +1,149 @@
|
||||
#!/usr/bin/env bash
|
||||
# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape
|
||||
# on Kunlun P800 (DeepSeek-V4-Flash-INT8).
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/config.env"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
||||
|
||||
ENGINE="sglang"
|
||||
ENGINE_PORT="$SGLANG_PORT"
|
||||
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
|
||||
ACTIVE_ENGINE_SERVER_LOG=""
|
||||
|
||||
PYTHON="${PYTHON:-$(command -v python3)}"
|
||||
|
||||
# The shared library's GPU monitor uses nvidia-smi; override it with an
|
||||
# xpu-smi sampler that emits the same CSV shape.
|
||||
adaptive_start_gpu_monitor() {
|
||||
local csv_path="$1"
|
||||
mkdir -p "$(dirname "$csv_path")"
|
||||
(
|
||||
echo "timestamp, index, memory.used [MiB], memory.total [MiB], utilization.gpu [%]"
|
||||
while true; do
|
||||
xpu-smi 2>/dev/null | awk -F'|' -v ts="$(date '+%Y/%m/%d %H:%M:%S')" '
|
||||
/^\|[[:space:]]*[0-9]+[[:space:]]/ {
|
||||
l=$0; sub(/^\|[[:space:]]*/, "", l); sub(/[[:space:]].*$/, "", l); idx=l
|
||||
}
|
||||
/MiB/ && $4 ~ /%/ {
|
||||
split($3, a, "/"); u=a[1]; t=a[2]
|
||||
gsub(/[^0-9.]/, "", u); gsub(/[^0-9.]/, "", t)
|
||||
r=$4; gsub(/[^0-9.]/, "", r)
|
||||
printf "%s, %s, %s MiB, %s MiB, %s %%\n", ts, idx, u, t, r
|
||||
}'
|
||||
sleep "$GPU_MEM_SAMPLE_INTERVAL_S"
|
||||
done
|
||||
) >> "$csv_path" 2>/dev/null &
|
||||
echo $!
|
||||
}
|
||||
|
||||
engine_is_healthy() {
|
||||
curl --fail --silent --show-error --max-time 5 \
|
||||
"http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
engine_stop_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
# The server only ever runs inside this experiment's dedicated container,
|
||||
# so removing the container is a complete and safe stop.
|
||||
log "stopping container ${CONTAINER_NAME} tp=${tp} dp=${dp}"
|
||||
docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
|
||||
ACTIVE_ENGINE_SERVER_LOG=""
|
||||
sleep 2
|
||||
}
|
||||
|
||||
engine_build_server_args() {
|
||||
# Keep in sync with the launch args in start_sglang_docker.sh.
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local args="--host 0.0.0.0 --port ${ENGINE_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH}"
|
||||
if (( dp > 1 )); then
|
||||
args="${args} --dp-size ${dp}"
|
||||
fi
|
||||
printf '%s' "$args"
|
||||
}
|
||||
|
||||
engine_start_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
|
||||
log "starting sglang server tp=${tp} dp=${dp}"
|
||||
bash "${SCRIPT_DIR}/start_sglang_docker.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||
if ! engine_is_healthy; then
|
||||
log "ERROR: sglang health check failed tp=${tp} dp=${dp}"
|
||||
return 1
|
||||
fi
|
||||
ACTIVE_ENGINE_SERVER_LOG=""
|
||||
log "sglang server healthy tp=${tp} dp=${dp} log=docker logs ${CONTAINER_NAME}"
|
||||
}
|
||||
|
||||
engine_detect_oom() {
|
||||
local detail_log="$1"
|
||||
local tp="$2"
|
||||
local dp="$3"
|
||||
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
|
||||
if grep -Eiq "$pattern" "$detail_log" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
docker logs --tail 2000 "$CONTAINER_NAME" 2>/dev/null | grep -Eiq "$pattern"
|
||||
}
|
||||
|
||||
engine_run_bench() {
|
||||
local isl="$1"
|
||||
local osl="$2"
|
||||
local concurrency="$3"
|
||||
local num_prompts="$4"
|
||||
local output_file="$5"
|
||||
local container_output="/tmp/bench_outputs/adaptive_$(basename "$output_file")"
|
||||
if [[ "$output_file" == "/dev/null" ]]; then
|
||||
container_output="/dev/null"
|
||||
fi
|
||||
|
||||
local -a bench_args=(
|
||||
--backend sglang
|
||||
--host 127.0.0.1
|
||||
--port "$ENGINE_PORT"
|
||||
--model "$MODEL_PATH"
|
||||
--dataset-name "$BENCH_DATASET_NAME"
|
||||
--random-input-len "$isl"
|
||||
--random-output-len "$osl"
|
||||
--random-range-ratio "$RANDOM_RANGE_RATIO"
|
||||
--num-prompts "$num_prompts"
|
||||
--max-concurrency "$concurrency"
|
||||
--request-rate 10000
|
||||
--output-file "$container_output"
|
||||
--output-details
|
||||
)
|
||||
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||
bench_args+=(--dataset-path "$CONTAINER_DATASET_PATH")
|
||||
else
|
||||
bench_args+=(--tokenize-prompt)
|
||||
fi
|
||||
|
||||
docker exec "$CONTAINER_NAME" mkdir -p /tmp/bench_outputs 2>/dev/null || true
|
||||
docker exec "$CONTAINER_NAME" \
|
||||
env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \
|
||||
"$CONTAINER_PYTHON" -m sglang.bench_serving "${bench_args[@]}" || return $?
|
||||
|
||||
if [[ "$output_file" != "/dev/null" ]]; then
|
||||
docker cp "$CONTAINER_NAME:${container_output}" "$output_file" || return 1
|
||||
fi
|
||||
}
|
||||
|
||||
export -f engine_run_bench
|
||||
export ENGINE_PORT CONTAINER_NAME CONTAINER_PYTHON MODEL_PATH RESULT_BASE
|
||||
export BENCH_DATASET_NAME CONTAINER_DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
|
||||
|
||||
adaptive_main "$@"
|
||||
152
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh
Executable file
152
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh
Executable file
@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env bash
|
||||
# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape
|
||||
# on Kunlun P800 (DeepSeek-V4-Flash-INT8).
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/config.env"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
||||
|
||||
ENGINE="sglang"
|
||||
ENGINE_PORT="$SGLANG_PORT"
|
||||
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
|
||||
ACTIVE_ENGINE_SERVER_LOG=""
|
||||
|
||||
PYTHON="${PYTHON:-$(command -v python3)}"
|
||||
|
||||
# The shared library's GPU monitor uses nvidia-smi; override it with an
|
||||
# xpu-smi sampler that emits the same CSV shape.
|
||||
adaptive_start_gpu_monitor() {
|
||||
local csv_path="$1"
|
||||
mkdir -p "$(dirname "$csv_path")"
|
||||
(
|
||||
echo "timestamp, index, memory.used [MiB], memory.total [MiB], utilization.gpu [%]"
|
||||
while true; do
|
||||
xpu-smi 2>/dev/null | awk -F'|' -v ts="$(date '+%Y/%m/%d %H:%M:%S')" '
|
||||
/^\|[[:space:]]*[0-9]+[[:space:]]/ {
|
||||
l=$0; sub(/^\|[[:space:]]*/, "", l); sub(/[[:space:]].*$/, "", l); idx=l
|
||||
}
|
||||
/MiB/ && $4 ~ /%/ {
|
||||
split($3, a, "/"); u=a[1]; t=a[2]
|
||||
gsub(/[^0-9.]/, "", u); gsub(/[^0-9.]/, "", t)
|
||||
r=$4; gsub(/[^0-9.]/, "", r)
|
||||
printf "%s, %s, %s MiB, %s MiB, %s %%\n", ts, idx, u, t, r
|
||||
}'
|
||||
sleep "$GPU_MEM_SAMPLE_INTERVAL_S"
|
||||
done
|
||||
) >> "$csv_path" 2>/dev/null &
|
||||
echo $!
|
||||
}
|
||||
|
||||
engine_is_healthy() {
|
||||
curl --fail --silent --show-error --max-time 5 \
|
||||
"http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
engine_stop_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
# The server only ever runs inside this experiment's dedicated container,
|
||||
# so removing the container is a complete and safe stop.
|
||||
log "stopping container ${CONTAINER_NAME} tp=${tp} dp=${dp}"
|
||||
docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
|
||||
ACTIVE_ENGINE_SERVER_LOG=""
|
||||
sleep 2
|
||||
}
|
||||
|
||||
engine_build_server_args() {
|
||||
# Keep in sync with the launch args in start_sglang_docker.sh.
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local args="--host 0.0.0.0 --port ${ENGINE_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH}"
|
||||
if (( dp > 1 )); then
|
||||
args="${args} --dp-size ${dp}"
|
||||
fi
|
||||
printf '%s' "$args"
|
||||
}
|
||||
|
||||
engine_start_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
|
||||
log "starting sglang server tp=${tp} dp=${dp}"
|
||||
bash "${SCRIPT_DIR}/start_sglang_docker.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||
if ! engine_is_healthy; then
|
||||
log "ERROR: sglang health check failed tp=${tp} dp=${dp}"
|
||||
return 1
|
||||
fi
|
||||
ACTIVE_ENGINE_SERVER_LOG=""
|
||||
log "sglang server healthy tp=${tp} dp=${dp} log=docker logs ${CONTAINER_NAME}"
|
||||
}
|
||||
|
||||
engine_detect_oom() {
|
||||
local detail_log="$1"
|
||||
local tp="$2"
|
||||
local dp="$3"
|
||||
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
|
||||
if grep -Eiq "$pattern" "$detail_log" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
docker logs --tail 2000 "$CONTAINER_NAME" 2>/dev/null | grep -Eiq "$pattern"
|
||||
}
|
||||
|
||||
engine_run_bench() {
|
||||
local isl="$1"
|
||||
local osl="$2"
|
||||
local concurrency="$3"
|
||||
local num_prompts="$4"
|
||||
local output_file="$5"
|
||||
local container_output="/tmp/bench_outputs/adaptive_$(basename "$output_file")"
|
||||
if [[ "$output_file" == "/dev/null" ]]; then
|
||||
container_output="/dev/null"
|
||||
fi
|
||||
|
||||
local -a bench_args=(
|
||||
--backend sglang
|
||||
--host 127.0.0.1
|
||||
--port "$ENGINE_PORT"
|
||||
--model "$MODEL_PATH"
|
||||
--dataset-name "$BENCH_DATASET_NAME"
|
||||
--random-input-len "$isl"
|
||||
--random-output-len "$osl"
|
||||
--random-range-ratio "$RANDOM_RANGE_RATIO"
|
||||
--num-prompts "$num_prompts"
|
||||
--max-concurrency "$concurrency"
|
||||
--request-rate 10000
|
||||
--output-file "$container_output"
|
||||
--output-details
|
||||
)
|
||||
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||
bench_args+=(--dataset-path "$CONTAINER_DATASET_PATH")
|
||||
else
|
||||
bench_args+=(--tokenize-prompt)
|
||||
fi
|
||||
|
||||
docker exec "$CONTAINER_NAME" mkdir -p /tmp/bench_outputs 2>/dev/null || true
|
||||
docker exec "$CONTAINER_NAME" \
|
||||
env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \
|
||||
"$CONTAINER_PYTHON" -m sglang.bench_serving "${bench_args[@]}" || return $?
|
||||
|
||||
if [[ "$output_file" != "/dev/null" ]]; then
|
||||
docker cp "$CONTAINER_NAME:${container_output}" "$output_file" || return 1
|
||||
fi
|
||||
}
|
||||
|
||||
export -f engine_run_bench
|
||||
export ENGINE_PORT CONTAINER_NAME CONTAINER_PYTHON MODEL_PATH RESULT_BASE
|
||||
export BENCH_DATASET_NAME CONTAINER_DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
|
||||
|
||||
export SEARCH_START_CONCURRENCY=16
|
||||
export SEARCH_ADDEND=16
|
||||
|
||||
adaptive_main "$@"
|
||||
555
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh
Executable file
555
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh
Executable file
@ -0,0 +1,555 @@
|
||||
#!/usr/bin/env bash
|
||||
# TP×DP matrix benchmark for DeepSeek-V4-Flash-INT8 on Kunlun P800 SGLang (Docker).
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/config.env"
|
||||
|
||||
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
|
||||
RESULT_BASE="${SCRIPT_DIR}/results"
|
||||
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}"
|
||||
MATRIX_MODE="${MATRIX_MODE:-Y}"
|
||||
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||||
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||||
DRY_RUN="${DRY_RUN:-0}"
|
||||
GRID_LIMIT="${GRID_LIMIT:-0}"
|
||||
|
||||
PYTHON="${PYTHON:-$(command -v python3)}"
|
||||
|
||||
# Export variables used inside functions that are called via bash -c subshells.
|
||||
export CONTAINER_NAME CONTAINER_PYTHON MODEL_PATH SGLANG_PORT CONTAINER_DATASET_PATH
|
||||
|
||||
log_dir_global="${RESULT_BASE}/${RUN_ID}/logs"
|
||||
mkdir -p "$log_dir_global"
|
||||
log_init "${log_dir_global}/orchestrator.log"
|
||||
|
||||
log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}"
|
||||
log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE} dry_run=${DRY_RUN} grid_limit=${GRID_LIMIT}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
is_server_healthy() {
|
||||
curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
container_running() {
|
||||
docker inspect "$CONTAINER_NAME" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
stop_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
|
||||
# The server only ever runs inside this experiment's dedicated container,
|
||||
# so removing the container is a complete and safe stop.
|
||||
if container_running; then
|
||||
log "stopping container ${CONTAINER_NAME} (tp=${tp}, dp=${dp})"
|
||||
docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
|
||||
else
|
||||
docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
|
||||
fi
|
||||
sleep 2
|
||||
}
|
||||
|
||||
build_server_args() {
|
||||
# Keep in sync with the launch args in start_sglang_docker.sh.
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
|
||||
local args="--host 0.0.0.0 --port ${SGLANG_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH}"
|
||||
if [[ "$dp" -gt 1 ]]; then
|
||||
args="${args} --dp-size ${dp}"
|
||||
fi
|
||||
printf '%s' "$args"
|
||||
}
|
||||
|
||||
start_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
|
||||
log "starting sglang server tp=${tp} dp=${dp}"
|
||||
bash "${SCRIPT_DIR}/start_sglang_docker.sh" "$tp" "$dp" \
|
||||
>> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1
|
||||
|
||||
if ! is_server_healthy; then
|
||||
log "error: sglang server tp=${tp} dp=${dp} failed health check on port ${SGLANG_PORT}"
|
||||
return 1
|
||||
fi
|
||||
log "sglang server tp=${tp} dp=${dp} is healthy on port ${SGLANG_PORT}"
|
||||
}
|
||||
|
||||
restart_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
log "restarting sglang server tp=${tp} dp=${dp} after non-OOM failure"
|
||||
stop_server "$tp" "$dp"
|
||||
sleep 10
|
||||
start_server "$tp" "$dp"
|
||||
}
|
||||
|
||||
run_bench_serving() {
|
||||
# Run sglang.bench_serving inside the server container (the P800 image
|
||||
# carries the only working client environment for this fork).
|
||||
docker exec "$CONTAINER_NAME" \
|
||||
env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \
|
||||
"$CONTAINER_PYTHON" -m sglang.bench_serving "$@"
|
||||
}
|
||||
export -f run_bench_serving
|
||||
|
||||
run_warmup() {
|
||||
local input_len="$1"
|
||||
local output_len="$2"
|
||||
|
||||
log "warming up (input=${input_len}, output=${output_len}, num=1)"
|
||||
bash -c '
|
||||
run_bench_serving \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port "'"$SGLANG_PORT"'" \
|
||||
--model "'"$MODEL_PATH"'" \
|
||||
--dataset-name random \
|
||||
--dataset-path "'"$CONTAINER_DATASET_PATH"'" \
|
||||
--random-input-len "'"$input_len"'" \
|
||||
--random-output-len "'"$output_len"'" \
|
||||
--random-range-ratio 1.0 \
|
||||
--num-prompts 1 \
|
||||
--max-concurrency 1 \
|
||||
--request-rate 10000 \
|
||||
--output-file /dev/null \
|
||||
--output-details \
|
||||
>> "'"${log_dir_global}/warmup.log"'" 2>&1
|
||||
' || log "WARNING: warmup failed; continuing with config"
|
||||
log "warmup completed"
|
||||
}
|
||||
|
||||
scenario_already_completed() {
|
||||
local output_file="$1"
|
||||
local expected="$2"
|
||||
[[ -s "$output_file" ]] || return 1
|
||||
local completed
|
||||
completed="$("$PYTHON" -c "
|
||||
import json, sys
|
||||
path = sys.argv[1]
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
data = json.loads(line)
|
||||
print(data.get('completed', 0))
|
||||
break
|
||||
except Exception:
|
||||
print(0)
|
||||
" "$output_file")"
|
||||
[[ "${completed:-0}" -ge "$expected" ]]
|
||||
}
|
||||
|
||||
scenario_already_processed() {
|
||||
local result_root="$1"
|
||||
local scenario_name="$2"
|
||||
local json_path="${result_root}/results.json"
|
||||
[[ -f "$json_path" ]] || return 1
|
||||
"$PYTHON" -c "
|
||||
import json, sys
|
||||
path, name = sys.argv[1], sys.argv[2]
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
for s in data.get('scenarios', []):
|
||||
if s.get('name') == name:
|
||||
if s.get('status') or s.get('metrics', {}).get('success', 0) > 0:
|
||||
sys.exit(0)
|
||||
except Exception:
|
||||
pass
|
||||
sys.exit(1)
|
||||
" "$json_path" "$scenario_name"
|
||||
}
|
||||
|
||||
detect_oom() {
|
||||
local detail_log="$1"
|
||||
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
|
||||
if grep -Eiq "$pattern" "$detail_log" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
docker logs --tail 2000 "$CONTAINER_NAME" 2>/dev/null | grep -Eiq "$pattern"
|
||||
}
|
||||
|
||||
start_gpu_monitor() {
|
||||
# Sample XPU memory/utilization with xpu-smi and emit nvidia-smi-style CSV
|
||||
# rows so scripts/common/parse_backend.py can read them unchanged.
|
||||
local csv_path="$1"
|
||||
mkdir -p "$(dirname "$csv_path")"
|
||||
(
|
||||
echo "timestamp, index, memory.used [MiB], memory.total [MiB], utilization.gpu [%]"
|
||||
while true; do
|
||||
xpu-smi 2>/dev/null | awk -F'|' -v ts="$(date '+%Y/%m/%d %H:%M:%S')" '
|
||||
/^\|[[:space:]]*[0-9]+[[:space:]]/ {
|
||||
l=$0; sub(/^\|[[:space:]]*/, "", l); sub(/[[:space:]].*$/, "", l); idx=l
|
||||
}
|
||||
/MiB/ && $4 ~ /%/ {
|
||||
split($3, a, "/"); u=a[1]; t=a[2]
|
||||
gsub(/[^0-9.]/, "", u); gsub(/[^0-9.]/, "", t)
|
||||
r=$4; gsub(/[^0-9.]/, "", r)
|
||||
printf "%s, %s, %s MiB, %s MiB, %s %%\n", ts, idx, u, t, r
|
||||
}'
|
||||
sleep "$GPU_MEM_SAMPLE_INTERVAL_S"
|
||||
done
|
||||
) >> "$csv_path" 2>/dev/null &
|
||||
echo $!
|
||||
}
|
||||
|
||||
stop_gpu_monitor() {
|
||||
local pid="$1"
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill "$pid" 2>/dev/null || true
|
||||
sleep 1
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
scenario_config_json() {
|
||||
"$PYTHON" -c "
|
||||
import json, sys
|
||||
print(json.dumps({
|
||||
'phase': 'main',
|
||||
'concurrency': int(sys.argv[1]),
|
||||
'input_len': int(sys.argv[2]),
|
||||
'output_len': int(sys.argv[3]),
|
||||
'dataset': 'random',
|
||||
'num_prompts': int(sys.argv[4]),
|
||||
}))
|
||||
" "$1" "$2" "$3" "$4"
|
||||
}
|
||||
|
||||
append_scenario_record() {
|
||||
local result_root="$1"
|
||||
local json_path="$result_root/results.json"
|
||||
shift
|
||||
local scenario_json
|
||||
scenario_json="$("$PYTHON" -c "
|
||||
import json, sys
|
||||
pairs = [a.split('=', 1) for a in sys.argv[1:]]
|
||||
d = {}
|
||||
for k, v in pairs:
|
||||
try:
|
||||
d[k] = json.loads(v)
|
||||
except json.JSONDecodeError:
|
||||
d[k] = v
|
||||
print(json.dumps(d, ensure_ascii=False))
|
||||
" "$@")"
|
||||
PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json"
|
||||
}
|
||||
|
||||
record_skipped_csv() {
|
||||
local csv_path="$1"
|
||||
shift
|
||||
# Args: key=value
|
||||
local row
|
||||
row="$("$PYTHON" -c "
|
||||
import csv, json, sys, io
|
||||
pairs = [a.split('=', 1) for a in sys.argv[1:]]
|
||||
d = {}
|
||||
for k, v in pairs:
|
||||
try:
|
||||
d[k] = json.loads(v)
|
||||
except json.JSONDecodeError:
|
||||
d[k] = v
|
||||
buf = io.StringIO()
|
||||
writer = csv.DictWriter(buf, fieldnames=['engine','tp','dp','mark','isl','osl','concurrency','status','reason','detail_log'], extrasaction='ignore')
|
||||
writer.writerow(d)
|
||||
print(buf.getvalue().strip())
|
||||
" "$@")"
|
||||
echo "$row" >> "$csv_path"
|
||||
}
|
||||
|
||||
skip_remaining_scenarios() {
|
||||
local result_root="$1"
|
||||
local scenario_tsv="$2"
|
||||
local start_index="$3"
|
||||
local status="$4"
|
||||
local reason="$5"
|
||||
local tp="$6"
|
||||
local dp="$7"
|
||||
local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv"
|
||||
|
||||
local i=0
|
||||
tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl osl conc num; do
|
||||
if (( i < start_index )); then
|
||||
i=$((i + 1))
|
||||
continue
|
||||
fi
|
||||
i=$((i + 1))
|
||||
local sname="c${conc}_i${isl}_o${osl}"
|
||||
if scenario_already_processed "$result_root" "$sname"; then
|
||||
continue
|
||||
fi
|
||||
append_scenario_record "$result_root" \
|
||||
"name=${sname}" \
|
||||
"config=$(scenario_config_json "$conc" "$isl" "$osl" "$num")" \
|
||||
"status=\"${status}\"" \
|
||||
"note=\"${reason}\""
|
||||
record_skipped_csv "$skipped_csv" \
|
||||
"engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=${status}" "reason=${reason}"
|
||||
done
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-configuration runner
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
run_parallel_config() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local config_label="tp${tp}_dp${dp}"
|
||||
local result_root="${RESULT_BASE}/${RUN_ID}/${config_label}"
|
||||
local raw_dir="${result_root}/raw_outputs"
|
||||
local gpu_log_dir="${result_root}/gpu_logs"
|
||||
local phase_log_dir="${result_root}/logs"
|
||||
mkdir -p "$raw_dir" "$gpu_log_dir" "$phase_log_dir"
|
||||
|
||||
log "===== ${config_label} START ====="
|
||||
|
||||
# Generate scenario list for this config.
|
||||
local scenario_tsv="${result_root}/scenarios.tsv"
|
||||
"$PYTHON" "${SCRIPT_DIR}/generate_scenarios.py" \
|
||||
--matrix "$MATRIX_FILE" \
|
||||
--mode "$MATRIX_MODE" \
|
||||
> "$scenario_tsv"
|
||||
local total_scenarios
|
||||
total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)"
|
||||
log "generated ${total_scenarios} scenarios for ${config_label}"
|
||||
|
||||
# Write metadata.
|
||||
ensure_result_root "$result_root"
|
||||
write_metadata_json \
|
||||
"${result_root}/results.json" \
|
||||
"${EXPERIMENT_NAME}_${config_label}" \
|
||||
"$RUN_ID" \
|
||||
"$MODEL_PATH" \
|
||||
"sglang" \
|
||||
"sglang-xpu" \
|
||||
"$HARDWARE" \
|
||||
"$ACCELERATOR" \
|
||||
"$CHIP" \
|
||||
"experiments/p800/${EXPERIMENT_NAME}/run_bench.sh" \
|
||||
"$DOCKER_IMAGE" \
|
||||
"P800 SGLang TP×DP matrix for DeepSeek-V4-Flash-INT8"
|
||||
|
||||
local server_args_str
|
||||
server_args_str="$(build_server_args "$tp" "$dp")"
|
||||
"$PYTHON" - "$tp" "$dp" "$server_args_str" "${result_root}/results.json" <<'PY'
|
||||
import json, os, sys
|
||||
|
||||
tp, dp, server_args, path = int(sys.argv[1]), int(sys.argv[2]), sys.argv[3], sys.argv[4]
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
data["config"] = {
|
||||
"tp": tp,
|
||||
"dp": dp,
|
||||
"xpu_visible_devices": os.environ.get("XPU_VISIBLE_DEVICES", "0,1,2,3,4,5,6,7"),
|
||||
"backend": "sglang",
|
||||
"engine": "sglang-xpu",
|
||||
"server_start_script": "experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_docker.sh",
|
||||
"server_args": server_args,
|
||||
}
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
PY
|
||||
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
log "DRY_RUN: would start server with args: ${server_args_str}"
|
||||
local line
|
||||
tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl osl conc num; do
|
||||
log "DRY_RUN: ${config_label} scenario mark=${mark} c=${conc} i=${isl} o=${osl} n=${num}"
|
||||
done
|
||||
log "===== ${config_label} DONE (dry run) ====="
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Initialize skipped_after_oom.csv for this run.
|
||||
local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv"
|
||||
if [[ ! -f "$skipped_csv" ]]; then
|
||||
echo "engine,tp,dp,mark,isl,osl,concurrency,status,reason,detail_log" > "$skipped_csv"
|
||||
fi
|
||||
|
||||
# Start server once for this TP×DP config.
|
||||
if ! start_server "$tp" "$dp"; then
|
||||
log "ERROR: ${config_label} failed to start; skipping all scenarios"
|
||||
skip_remaining_scenarios "$result_root" "$scenario_tsv" 0 "SKIPPED_SERVICE_START_FAILED" "service failed to start" "$tp" "$dp"
|
||||
log "===== ${config_label} DONE ====="
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Warmup with a small prompt before the first scenario.
|
||||
run_warmup 1024 128 || true
|
||||
|
||||
# Read scenarios into an array so we can skip remaining entries on failure.
|
||||
local -a scenarios=()
|
||||
while IFS= read -r line; do
|
||||
scenarios+=("$line")
|
||||
done < <(tail -n +2 "$scenario_tsv")
|
||||
|
||||
local i mark isl osl conc num
|
||||
local output_file container_output detail_log gpu_csv sname bench_rc
|
||||
for (( i = 0; i < ${#scenarios[@]}; i++ )); do
|
||||
IFS=$'\t' read -r mark isl osl conc num <<< "${scenarios[$i]}"
|
||||
|
||||
if [[ "$GRID_LIMIT" -gt 0 && "$i" -ge "$GRID_LIMIT" ]]; then
|
||||
log "GRID_LIMIT=${GRID_LIMIT} reached; skipping remaining scenarios"
|
||||
skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_GRID_LIMIT" "GRID_LIMIT reached" "$tp" "$dp"
|
||||
break
|
||||
fi
|
||||
|
||||
sname="c${conc}_i${isl}_o${osl}"
|
||||
output_file="${raw_dir}/sglang_main_${conc}_${isl}_${osl}.jsonl"
|
||||
container_output="/tmp/bench_outputs/sglang_main_${conc}_${isl}_${osl}.jsonl"
|
||||
detail_log="${phase_log_dir}/sglang_${config_label}_${sname}.log"
|
||||
gpu_csv="${gpu_log_dir}/gpu_mem_${conc}_${isl}_${osl}.csv"
|
||||
|
||||
if scenario_already_completed "$output_file" "$num" || scenario_already_processed "$result_root" "$sname"; then
|
||||
log "skipping already-processed ${config_label} scenario: ${sname}"
|
||||
continue
|
||||
fi
|
||||
|
||||
if ! container_running; then
|
||||
log "WARNING: container ${CONTAINER_NAME} died, skipping remaining scenarios in ${config_label}"
|
||||
skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_CONTAINER_DIED" "container died" "$tp" "$dp"
|
||||
break
|
||||
fi
|
||||
|
||||
log "running ${config_label} scenario: mark=${mark} c=${conc} i=${isl} o=${osl} n=${num}"
|
||||
|
||||
local gpu_pid
|
||||
gpu_pid="$(start_gpu_monitor "$gpu_csv")"
|
||||
|
||||
bench_rc=0
|
||||
docker exec "$CONTAINER_NAME" mkdir -p /tmp/bench_outputs
|
||||
timeout "$SCENARIO_TIMEOUT_S" bash -c '
|
||||
run_bench_serving \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port "'"$SGLANG_PORT"'" \
|
||||
--model "'"$MODEL_PATH"'" \
|
||||
--dataset-name random \
|
||||
--dataset-path "'"$CONTAINER_DATASET_PATH"'" \
|
||||
--random-input-len "'"$isl"'" \
|
||||
--random-output-len "'"$osl"'" \
|
||||
--random-range-ratio 1.0 \
|
||||
--num-prompts "'"$num"'" \
|
||||
--max-concurrency "'"$conc"'" \
|
||||
--request-rate 10000 \
|
||||
--output-file "'"$container_output"'" \
|
||||
--output-details \
|
||||
> "'"$detail_log"'" 2>&1
|
||||
' || bench_rc=$?
|
||||
|
||||
stop_gpu_monitor "$gpu_pid"
|
||||
|
||||
if [[ "$bench_rc" -eq 0 ]]; then
|
||||
if docker cp "${CONTAINER_NAME}:${container_output}" "$output_file"; then
|
||||
log "finished ${config_label} scenario: output=${output_file}"
|
||||
append_scenario_record "$result_root" \
|
||||
"name=${sname}" \
|
||||
"config=$(scenario_config_json "$conc" "$isl" "$osl" "$num")" \
|
||||
"status=\"completed\"" \
|
||||
"note=\"benchmark finished successfully\""
|
||||
continue
|
||||
fi
|
||||
log "ERROR: failed to copy ${container_output} from container"
|
||||
bench_rc=1
|
||||
fi
|
||||
|
||||
# Failure handling.
|
||||
if detect_oom "$detail_log"; then
|
||||
log "ERROR: ${config_label} scenario ${sname} triggered OOM; stopping config"
|
||||
append_scenario_record "$result_root" \
|
||||
"name=${sname}" \
|
||||
"config=$(scenario_config_json "$conc" "$isl" "$osl" "$num")" \
|
||||
"status=\"OOM\"" \
|
||||
"note=\"detected out-of-memory\""
|
||||
record_skipped_csv "$skipped_csv" \
|
||||
"engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=OOM" "reason=detected out-of-memory" "detail_log=${detail_log}"
|
||||
stop_server "$tp" "$dp"
|
||||
skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_AFTER_OOM" "previous case OOM" "$tp" "$dp"
|
||||
break
|
||||
fi
|
||||
|
||||
log "ERROR: ${config_label} scenario ${sname} failed (rc=${bench_rc}); see ${detail_log}"
|
||||
if [[ "$mark" == "P" ]]; then
|
||||
log "optional (P) scenario failed; recording as skipped and continuing"
|
||||
append_scenario_record "$result_root" \
|
||||
"name=${sname}" \
|
||||
"config=$(scenario_config_json "$conc" "$isl" "$osl" "$num")" \
|
||||
"status=\"skipped_optional\"" \
|
||||
"note=\"optional scenario failed (rc=${bench_rc})\""
|
||||
record_skipped_csv "$skipped_csv" \
|
||||
"engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=skipped_optional" "reason=optional scenario failed (rc=${bench_rc})" "detail_log=${detail_log}"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Mandatory scenario failed but not OOM: try to restart the server.
|
||||
if restart_server "$tp" "$dp"; then
|
||||
run_warmup 1024 128 || true
|
||||
log "resuming ${config_label} after server restart"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "ERROR: ${config_label} server restart failed; skipping remaining scenarios"
|
||||
append_scenario_record "$result_root" \
|
||||
"name=${sname}" \
|
||||
"config=$(scenario_config_json "$conc" "$isl" "$osl" "$num")" \
|
||||
"status=\"FAILED\"" \
|
||||
"note=\"scenario failed and server restart failed (rc=${bench_rc})\""
|
||||
record_skipped_csv "$skipped_csv" \
|
||||
"engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=FAILED" "reason=scenario failed and server restart failed" "detail_log=${detail_log}"
|
||||
skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_RESTART_FAILED" "server restart failed" "$tp" "$dp"
|
||||
break
|
||||
done
|
||||
|
||||
stop_server "$tp" "$dp"
|
||||
|
||||
# Parse results.
|
||||
log "parsing ${config_label} results"
|
||||
"$PYTHON" "${SCRIPT_DIR}/../../../scripts/common/parse_backend.py" "$result_root" --backend sglang \
|
||||
>> "${phase_log_dir}/parse.log" 2>&1 || {
|
||||
log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log"
|
||||
}
|
||||
|
||||
log "===== ${config_label} DONE ====="
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Cleanup any leftovers.
|
||||
for cfg in "${PARALLEL_CONFIGS[@]}"; do
|
||||
read -r tp dp <<< "$cfg"
|
||||
stop_server "$tp" "$dp"
|
||||
done
|
||||
|
||||
# Run each parallel configuration.
|
||||
for cfg in "${PARALLEL_CONFIGS[@]}"; do
|
||||
read -r tp dp <<< "$cfg"
|
||||
run_parallel_config "$tp" "$dp"
|
||||
done
|
||||
|
||||
# Generate cross-configuration comparison.
|
||||
log "generating comparison report"
|
||||
"$PYTHON" "${SCRIPT_DIR}/compare.py" \
|
||||
--run-root "${RESULT_BASE}/${RUN_ID}" \
|
||||
--output "${RESULT_BASE}/${RUN_ID}/comparison.md" \
|
||||
>> "${log_dir_global}/compare.log" 2>&1 || {
|
||||
log "WARNING: comparison script failed; see ${log_dir_global}/compare.log"
|
||||
}
|
||||
|
||||
log "all results saved to ${RESULT_BASE}/${RUN_ID}"
|
||||
161
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_docker.sh
Executable file
161
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_docker.sh
Executable file
@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env bash
|
||||
# Start the P800 SGLang INT8 server in Docker for a given TP×DP configuration.
|
||||
# Usage: start_sglang_docker.sh <TP> <DP>
|
||||
#
|
||||
# Based on the proven P800 INT8 launch from dsv4_p800_long_context_matrix,
|
||||
# with --tensor-parallel-size/--dp-size parameterized per config.
|
||||
set -Eeuo pipefail
|
||||
|
||||
TP="${1:-}"
|
||||
DP="${2:-}"
|
||||
if [[ -z "$TP" || -z "$DP" ]]; then
|
||||
echo "Usage: $0 <TP> <DP>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/config.env"
|
||||
|
||||
PORT="${SGLANG_PORT:-30014}"
|
||||
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||
mkdir -p "${RUNTIME_BASE}/logs"
|
||||
SERVER_LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
|
||||
|
||||
log "starting P800 SGLang INT8 server (tp=${TP}, dp=${DP})"
|
||||
log "model: ${MODEL_PATH}"
|
||||
log "port: ${PORT}"
|
||||
log "container: ${CONTAINER_NAME}"
|
||||
|
||||
# Stop any existing container with the same name.
|
||||
docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
|
||||
|
||||
# Build device args.
|
||||
device_args=""
|
||||
for i in 0 1 2 3 4 5 6 7; do
|
||||
device_args="${device_args} --device /dev/xpu${i}:/dev/xpu${i}"
|
||||
done
|
||||
device_args="${device_args} --device /dev/xpuctrl:/dev/xpuctrl"
|
||||
|
||||
# Environment variables required by the P800 SGLang INT8 image.
|
||||
env_args=(
|
||||
-e XPU_VISIBLE_DEVICES="${XPU_VISIBLE_DEVICES}"
|
||||
-e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}"
|
||||
-e CUDA_DEVICE_ORDER=OAM_ID
|
||||
-e SGLANG_USE_TRANSFORMERS_V5_TOKENIZER=1
|
||||
-e XMLIR_FORCE_USE_XPU_GRAPH=1
|
||||
-e SGLANG_DSV4_MODE=2604
|
||||
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
-e SGLANG_NSA_DUAL_STREAM=true
|
||||
-e SGLANG_NSA_QUANT_WQ_B_WK=false
|
||||
-e SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1
|
||||
-e SGLANG_OPT_DEEPGEMM_HC_PRENORM=false
|
||||
-e SGLANG_OPT_USE_TILELANG_MHC_PRE=1
|
||||
-e SGLANG_OPT_USE_TILELANG_MHC_POST=1
|
||||
-e SGLANG_CLEAN_REQUEST_WHEN_RETRACT=1
|
||||
-e SGLANG_SET_CPU_AFFINITY=1
|
||||
-e SGLANG_OPT_USE_KLX_TOPK_KERNEL=1
|
||||
-e XSGL_INTERTYPE_BFP16=1
|
||||
-e ENABLE_FAST_BFP16_ATTN=1
|
||||
-e XSGL_USE_DEEP_GEMM_BMM=1
|
||||
-e XSGL_XDNN_QUANT=1
|
||||
-e XSGL_FUSE_RMS_NORM_QUANT=1
|
||||
-e XSGL_TRANSPOSE_MATMUL_WEIGHT=1
|
||||
-e XINFER_QUANT_SDNN=1
|
||||
-e XSGL_USE_MOE_SIGMOID_GROUP_TOPK_NORM=1
|
||||
-e XSGL_EARLY_FIRST_TOKEN=1
|
||||
-e XSGL_ENABLE_TGEMM_FP16=1
|
||||
-e SGLANG_ENABLE_SPEC_V2=True
|
||||
-e SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
-e PYTHONDONTWRITEBYTECODE=1
|
||||
-e XTORCH_OPS_LIB_DIR=/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/xtorch_ops
|
||||
-e XPU_RUNTIME_LIB_DIR=/root/miniconda/envs/python310_torch25_cuda/xcudart/lib
|
||||
-e BKCL_TREE_THRESHOLD=1048576
|
||||
-e CUDA_ENABLE_P2P_NO_UVA=1
|
||||
-e NCCL_IB_GID_INDEX=3
|
||||
-e IS_DSV4=1
|
||||
-e MC_CUSTOM_TOPO_JSON=/workspace/nic_priority_matrix_test.json
|
||||
# INT8 specific
|
||||
-e SGLANG_DSV4_FP4_EXPERTS=false
|
||||
-e SGLANG_APPLY_CONFIG_BACKUP=auto
|
||||
-e BKCL_ENABLE_XDR=1
|
||||
-e BKCL_RDMA_NICS=eth1,eth1,eth3,eth3,eth5,eth5,eth7,eth7
|
||||
-e BKCL_RDMA_VERBS=1
|
||||
-e XSGL_INT8_LM_HEAD=1
|
||||
-e SGLANG_P800_ALL_GATHER_FALLBACK=0
|
||||
)
|
||||
|
||||
# Launch args. Same proven P800 INT8 command as
|
||||
# dsv4_p800_long_context_matrix/start_sglang.sh, with TP/DP parameterized.
|
||||
# Note: whether this image supports --dp-size on XPU is part of what this
|
||||
# experiment probes; TP=8/DP=1 matches the production configuration.
|
||||
launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${TP} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH}"
|
||||
|
||||
if [[ "$DP" -gt 1 ]]; then
|
||||
launch_args="${launch_args} --dp-size ${DP}"
|
||||
fi
|
||||
|
||||
# Base64-encode the bootstrap command to avoid host-shell quoting issues.
|
||||
server_cmd=$(cat <<EOF
|
||||
cd /workspace
|
||||
find /root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/sglang -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
|
||||
/root/miniconda/envs/python310_torch25_cuda/bin/pip install --upgrade safetensors -q
|
||||
/root/miniconda/envs/python310_torch25_cuda/bin/pip install https://files.pythonhosted.org/packages/14/8b/2a1333a6455c6fad401c2285dee6f58016c55b1cb44cae3a31f8a9cc7d83/apache_tvm_ffi-0.1.0b2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl -q
|
||||
/root/miniconda/envs/python310_torch25_cuda/bin/python -c "import torch; torch.float8_e8m0fnu = torch.uint8; import runpy, sys; sys.argv[0] = 'sglang.launch_server'; runpy.run_module('sglang.launch_server', run_name='__main__')" ${launch_args}
|
||||
EOF
|
||||
)
|
||||
server_cmd_b64=$(printf '%s' "$server_cmd" | base64 -w0)
|
||||
|
||||
patch_mounts=(
|
||||
-v "${PATCH_ROOT}/nic_priority_matrix_test.json:/workspace/nic_priority_matrix_test.json:ro"
|
||||
-v "${PATCH_ROOT}/dummy_sharegpt.json:/workspace/dummy_sharegpt.json:ro"
|
||||
)
|
||||
|
||||
echo "=== Starting P800 SGLang INT8 server in Docker (TP=${TP}, DP=${DP}) ==="
|
||||
echo "Image: ${DOCKER_IMAGE}"
|
||||
echo "Model: ${MODEL_PATH}"
|
||||
echo "Container name: ${CONTAINER_NAME}"
|
||||
echo "Host port: ${PORT}"
|
||||
echo "Command: sglang.launch_server ${launch_args}"
|
||||
echo "Log: docker logs ${CONTAINER_NAME}"
|
||||
|
||||
docker run -d \
|
||||
--name "${CONTAINER_NAME}" \
|
||||
--privileged \
|
||||
--network host \
|
||||
--ipc host \
|
||||
${device_args} \
|
||||
-v "${MODEL_PATH}:/models:ro" \
|
||||
-v "${MODEL_PATH}:${MODEL_PATH}:ro" \
|
||||
"${patch_mounts[@]}" \
|
||||
"${env_args[@]}" \
|
||||
"${DOCKER_IMAGE}" \
|
||||
bash -c "echo '${server_cmd_b64}' | base64 -d | bash" \
|
||||
>> "${SERVER_LOG}" 2>&1
|
||||
|
||||
log "container ${CONTAINER_NAME} started, waiting for health"
|
||||
healthy=0
|
||||
for ((i = 1; i <= 600; i++)); do
|
||||
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
|
||||
healthy=1
|
||||
break
|
||||
fi
|
||||
if [[ "$(docker inspect -f '{{.State.Running}}' "${CONTAINER_NAME}" 2>/dev/null)" != "true" ]]; then
|
||||
log "ERROR: container ${CONTAINER_NAME} exited during startup; last server logs:"
|
||||
docker logs --tail 100 "${CONTAINER_NAME}" 2>&1 || true
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [[ "$healthy" == "1" ]]; then
|
||||
log "container ${CONTAINER_NAME} is healthy"
|
||||
exit 0
|
||||
else
|
||||
log "ERROR: container ${CONTAINER_NAME} failed health check; last server logs:"
|
||||
docker logs --tail 100 "${CONTAINER_NAME}" 2>&1 || true
|
||||
exit 1
|
||||
fi
|
||||
11
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_dp.sh
Executable file
11
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_dp.sh
Executable file
@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
# Start SGLang server for a given TP×DP configuration.
|
||||
# Usage: start_sglang_dp.sh <TP> <DP>
|
||||
#
|
||||
# P800 SGLang always runs in the vendor Docker image, so this simply
|
||||
# delegates to the Docker start script. Kept under the same name as the
|
||||
# H20 experiment for parity.
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
exec "${SCRIPT_DIR}/start_sglang_docker.sh" "$@"
|
||||
@ -134,40 +134,42 @@ adaptive_append_point_from_metrics() {
|
||||
local gain_json
|
||||
gain_json="$(adaptive_json_number_or_null "$gain_pct")"
|
||||
|
||||
jq -c \
|
||||
--arg timestamp "$(adaptive_timestamp)" \
|
||||
--arg engine "$ENGINE" \
|
||||
--argjson tp "$tp" \
|
||||
--argjson dp "$dp" \
|
||||
--arg mark "$mark" \
|
||||
--argjson isl "$isl" \
|
||||
--argjson osl "$osl" \
|
||||
--argjson concurrency "$concurrency" \
|
||||
--argjson num_prompts "$num_prompts" \
|
||||
--argjson warmup_requests "$(adaptive_warmup_request_count "$concurrency")" \
|
||||
--argjson attempt "$attempt" \
|
||||
--argjson gain_pct "$gain_json" \
|
||||
--argjson plateau_streak "$plateau_streak" \
|
||||
--arg raw_file "$raw_file" \
|
||||
--arg detail_log "$detail_log" \
|
||||
'. + {
|
||||
timestamp: $timestamp,
|
||||
engine: $engine,
|
||||
tp: $tp,
|
||||
dp: $dp,
|
||||
mark: $mark,
|
||||
isl: $isl,
|
||||
osl: $osl,
|
||||
concurrency: $concurrency,
|
||||
num_prompts: $num_prompts,
|
||||
warmup_requests: $warmup_requests,
|
||||
attempt: $attempt,
|
||||
gain_pct: $gain_pct,
|
||||
plateau_streak: $plateau_streak,
|
||||
error_type: "",
|
||||
raw_file: $raw_file,
|
||||
detail_log: $detail_log
|
||||
}' "$metrics_file" >> "$ADAPTIVE_POINTS_JSONL"
|
||||
"$PYTHON" - \
|
||||
"$metrics_file" \
|
||||
"$(adaptive_timestamp)" \
|
||||
"$ENGINE" \
|
||||
"$tp" "$dp" "$mark" "$isl" "$osl" "$concurrency" "$num_prompts" \
|
||||
"$(adaptive_warmup_request_count "$concurrency")" \
|
||||
"$attempt" "$gain_json" "$plateau_streak" "$raw_file" "$detail_log" \
|
||||
>> "$ADAPTIVE_POINTS_JSONL" <<'PY'
|
||||
import json, sys
|
||||
|
||||
(path, timestamp, engine, tp, dp, mark, isl, osl, concurrency,
|
||||
num_prompts, warmup_requests, attempt, gain_json, plateau_streak,
|
||||
raw_file, detail_log) = sys.argv[1:17]
|
||||
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
data.update({
|
||||
"timestamp": timestamp,
|
||||
"engine": engine,
|
||||
"tp": int(tp),
|
||||
"dp": int(dp),
|
||||
"mark": mark,
|
||||
"isl": int(isl),
|
||||
"osl": int(osl),
|
||||
"concurrency": int(concurrency),
|
||||
"num_prompts": int(num_prompts),
|
||||
"warmup_requests": int(warmup_requests),
|
||||
"attempt": int(attempt),
|
||||
"gain_pct": json.loads(gain_json),
|
||||
"plateau_streak": int(plateau_streak),
|
||||
"error_type": "",
|
||||
"raw_file": raw_file,
|
||||
"detail_log": detail_log,
|
||||
})
|
||||
print(json.dumps(data, separators=(",", ":"), ensure_ascii=False))
|
||||
PY
|
||||
}
|
||||
|
||||
adaptive_append_failed_point() {
|
||||
@ -184,44 +186,42 @@ adaptive_append_failed_point() {
|
||||
local raw_file="${11}"
|
||||
local detail_log="${12}"
|
||||
|
||||
jq -nc \
|
||||
--arg timestamp "$(adaptive_timestamp)" \
|
||||
--arg engine "$ENGINE" \
|
||||
--argjson tp "$tp" \
|
||||
--argjson dp "$dp" \
|
||||
--arg mark "$mark" \
|
||||
--argjson isl "$isl" \
|
||||
--argjson osl "$osl" \
|
||||
--argjson concurrency "$concurrency" \
|
||||
--argjson num_prompts "$num_prompts" \
|
||||
--argjson warmup_requests "$(adaptive_warmup_request_count "$concurrency")" \
|
||||
--argjson attempt "$attempt" \
|
||||
--arg status "$status" \
|
||||
--arg error_type "$error_type" \
|
||||
--arg raw_file "$raw_file" \
|
||||
--arg detail_log "$detail_log" \
|
||||
'{
|
||||
timestamp: $timestamp,
|
||||
engine: $engine,
|
||||
tp: $tp,
|
||||
dp: $dp,
|
||||
mark: $mark,
|
||||
isl: $isl,
|
||||
osl: $osl,
|
||||
concurrency: $concurrency,
|
||||
num_prompts: $num_prompts,
|
||||
warmup_requests: $warmup_requests,
|
||||
attempt: $attempt,
|
||||
status: $status,
|
||||
completed: 0,
|
||||
failed: $num_prompts,
|
||||
error_type: $error_type,
|
||||
validation_errors: [],
|
||||
gain_pct: null,
|
||||
plateau_streak: 0,
|
||||
raw_file: $raw_file,
|
||||
detail_log: $detail_log
|
||||
}' >> "$ADAPTIVE_POINTS_JSONL"
|
||||
"$PYTHON" - \
|
||||
"$(adaptive_timestamp)" \
|
||||
"$ENGINE" \
|
||||
"$tp" "$dp" "$mark" "$isl" "$osl" "$concurrency" "$num_prompts" \
|
||||
"$(adaptive_warmup_request_count "$concurrency")" \
|
||||
"$attempt" "$status" "$error_type" "$raw_file" "$detail_log" \
|
||||
>> "$ADAPTIVE_POINTS_JSONL" <<'PY'
|
||||
import json, sys
|
||||
|
||||
(timestamp, engine, tp, dp, mark, isl, osl, concurrency, num_prompts,
|
||||
warmup_requests, attempt, status, error_type, raw_file,
|
||||
detail_log) = sys.argv[1:16]
|
||||
|
||||
print(json.dumps({
|
||||
"timestamp": timestamp,
|
||||
"engine": engine,
|
||||
"tp": int(tp),
|
||||
"dp": int(dp),
|
||||
"mark": mark,
|
||||
"isl": int(isl),
|
||||
"osl": int(osl),
|
||||
"concurrency": int(concurrency),
|
||||
"num_prompts": int(num_prompts),
|
||||
"warmup_requests": int(warmup_requests),
|
||||
"attempt": int(attempt),
|
||||
"status": status,
|
||||
"completed": 0,
|
||||
"failed": int(num_prompts),
|
||||
"error_type": error_type,
|
||||
"validation_errors": [],
|
||||
"gain_pct": None,
|
||||
"plateau_streak": 0,
|
||||
"raw_file": raw_file,
|
||||
"detail_log": detail_log,
|
||||
}, separators=(",", ":"), ensure_ascii=False))
|
||||
PY
|
||||
}
|
||||
|
||||
adaptive_append_shape_summary() {
|
||||
@ -246,43 +246,40 @@ adaptive_append_shape_summary() {
|
||||
best_tps_json="$(adaptive_json_number_or_null "$best_total_tps")"
|
||||
last_tps_json="$(adaptive_json_number_or_null "$last_total_tps")"
|
||||
|
||||
jq -nc \
|
||||
--arg timestamp "$(adaptive_timestamp)" \
|
||||
--arg engine "$ENGINE" \
|
||||
--argjson tp "$tp" \
|
||||
--argjson dp "$dp" \
|
||||
--arg mark "$mark" \
|
||||
--argjson isl "$isl" \
|
||||
--argjson osl "$osl" \
|
||||
--arg status "$status" \
|
||||
--arg stop_reason "$stop_reason" \
|
||||
--argjson tested_points "$tested_points" \
|
||||
--argjson search_cap "$SEARCH_MAX_CONCURRENCY" \
|
||||
--argjson max_successful_concurrency "$max_successful_concurrency" \
|
||||
--argjson saturation_concurrency "$saturation_json" \
|
||||
--argjson stop_probe_concurrency "$stop_probe_json" \
|
||||
--argjson best_tps_concurrency "$best_c_json" \
|
||||
--argjson best_total_tps "$best_tps_json" \
|
||||
--argjson last_total_tps "$last_tps_json" \
|
||||
'{
|
||||
timestamp: $timestamp,
|
||||
engine: $engine,
|
||||
tp: $tp,
|
||||
dp: $dp,
|
||||
mark: $mark,
|
||||
isl: $isl,
|
||||
osl: $osl,
|
||||
status: $status,
|
||||
stop_reason: $stop_reason,
|
||||
tested_points: $tested_points,
|
||||
search_cap: $search_cap,
|
||||
max_successful_concurrency: $max_successful_concurrency,
|
||||
saturation_concurrency: $saturation_concurrency,
|
||||
stop_probe_concurrency: $stop_probe_concurrency,
|
||||
best_tps_concurrency: $best_tps_concurrency,
|
||||
best_total_tps: $best_total_tps,
|
||||
last_total_tps: $last_total_tps
|
||||
}' >> "$ADAPTIVE_SHAPES_JSONL"
|
||||
"$PYTHON" - \
|
||||
"$(adaptive_timestamp)" \
|
||||
"$ENGINE" \
|
||||
"$tp" "$dp" "$mark" "$isl" "$osl" "$status" "$stop_reason" \
|
||||
"$tested_points" "$SEARCH_MAX_CONCURRENCY" "$max_successful_concurrency" \
|
||||
"$saturation_json" "$stop_probe_json" "$best_c_json" "$best_tps_json" "$last_tps_json" \
|
||||
>> "$ADAPTIVE_SHAPES_JSONL" <<'PY'
|
||||
import json, sys
|
||||
|
||||
(timestamp, engine, tp, dp, mark, isl, osl, status, stop_reason,
|
||||
tested_points, search_cap, max_successful_concurrency,
|
||||
saturation_json, stop_probe_json, best_c_json, best_tps_json,
|
||||
last_tps_json) = sys.argv[1:18]
|
||||
|
||||
print(json.dumps({
|
||||
"timestamp": timestamp,
|
||||
"engine": engine,
|
||||
"tp": int(tp),
|
||||
"dp": int(dp),
|
||||
"mark": mark,
|
||||
"isl": int(isl),
|
||||
"osl": int(osl),
|
||||
"status": status,
|
||||
"stop_reason": stop_reason,
|
||||
"tested_points": int(tested_points),
|
||||
"search_cap": int(search_cap),
|
||||
"max_successful_concurrency": int(max_successful_concurrency),
|
||||
"saturation_concurrency": json.loads(saturation_json),
|
||||
"stop_probe_concurrency": json.loads(stop_probe_json),
|
||||
"best_tps_concurrency": json.loads(best_c_json),
|
||||
"best_total_tps": json.loads(best_tps_json),
|
||||
"last_total_tps": json.loads(last_tps_json),
|
||||
}, separators=(",", ":"), ensure_ascii=False))
|
||||
PY
|
||||
}
|
||||
|
||||
adaptive_run_point() {
|
||||
@ -389,7 +386,7 @@ adaptive_run_shape() {
|
||||
stop_probe_concurrency="$concurrency"
|
||||
|
||||
if (( point_rc == 0 )); then
|
||||
current_tps="$(jq -r '.total_tps' "$POINT_METRICS_FILE")"
|
||||
current_tps="$("$PYTHON" -c 'import json, sys; print(json.load(open(sys.argv[1]))["total_tps"])' "$POINT_METRICS_FILE")"
|
||||
gain_pct=""
|
||||
meaningful_gain=1
|
||||
if [[ -n "$previous_tps" ]]; then
|
||||
@ -420,7 +417,7 @@ adaptive_run_shape() {
|
||||
# TTFT SLO check: stop searching if TTFT P95 exceeds the SLO threshold
|
||||
local ttft_p95_ms=""
|
||||
if [[ "${ENABLE_TTFT_SLO_STOP:-1}" == "1" ]]; then
|
||||
ttft_p95_ms="$(jq -r '.ttft_p95_ms' "$POINT_METRICS_FILE")"
|
||||
ttft_p95_ms="$("$PYTHON" -c 'import json, sys; print(json.load(open(sys.argv[1]))["ttft_p95_ms"])' "$POINT_METRICS_FILE")"
|
||||
if awk -v ttft="$ttft_p95_ms" -v slo="${TTFT_SLO_MS:-4000}" 'BEGIN { exit !(ttft > slo) }'; then
|
||||
adaptive_append_point_from_metrics \
|
||||
"$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \
|
||||
@ -543,9 +540,26 @@ adaptive_shape_already_tested() {
|
||||
return 1
|
||||
fi
|
||||
local count
|
||||
count="$(jq -s -r --argjson tp "$tp" --argjson dp "$dp" --arg mark "$mark" --argjson isl "$isl" --argjson osl "$osl" '
|
||||
[ .[] | select(.tp == $tp and .dp == $dp and .mark == $mark and .isl == $isl and .osl == $osl) ] | length
|
||||
' "$resume_shapes_jsonl" 2>/dev/null)"
|
||||
count="$("$PYTHON" - "$tp" "$dp" "$mark" "$isl" "$osl" "$resume_shapes_jsonl" 2>/dev/null <<'PY'
|
||||
import json, sys
|
||||
|
||||
tp, dp, mark, isl, osl = int(sys.argv[1]), int(sys.argv[2]), sys.argv[3], int(sys.argv[4]), int(sys.argv[5])
|
||||
count = 0
|
||||
with open(sys.argv[6], "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
d = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if d.get("tp") == tp and d.get("dp") == dp and d.get("mark") == mark \
|
||||
and d.get("isl") == isl and d.get("osl") == osl:
|
||||
count += 1
|
||||
print(count)
|
||||
PY
|
||||
)"
|
||||
if [[ -z "$count" ]]; then
|
||||
return 1
|
||||
fi
|
||||
@ -633,47 +647,56 @@ adaptive_main() {
|
||||
log "search_start=${SEARCH_START_CONCURRENCY} search_cap=${SEARCH_MAX_CONCURRENCY} multiplier=${SEARCH_MULTIPLIER} min_gain_pct=${TPS_MIN_GAIN_PCT} patience=${PLATEAU_PATIENCE} prompts_multiplier=${NUM_PROMPTS_MULTIPLIER} warmup_cap=${BENCH_WARMUP_MAX_REQUESTS}"
|
||||
log "dataset=${BENCH_DATASET_NAME} random_range_ratio=${RANDOM_RANGE_RATIO} tokenize_prompt=${tokenize_prompt_json} matrix=${MATRIX_FILE} mode=${MATRIX_MODE}"
|
||||
|
||||
jq -n \
|
||||
--arg experiment "$EXPERIMENT_NAME" \
|
||||
--arg engine "$ENGINE" \
|
||||
--arg run_id "$RUN_ID" \
|
||||
--arg model "$MODEL_PATH" \
|
||||
--arg hardware "$HARDWARE" \
|
||||
--arg matrix "$MATRIX_FILE" \
|
||||
--arg dataset "$BENCH_DATASET_NAME" \
|
||||
--argjson tokenize_prompt "$tokenize_prompt_json" \
|
||||
--argjson random_range_ratio "$RANDOM_RANGE_RATIO" \
|
||||
--argjson search_start "$SEARCH_START_CONCURRENCY" \
|
||||
--argjson search_cap "$SEARCH_MAX_CONCURRENCY" \
|
||||
--argjson search_multiplier "$SEARCH_MULTIPLIER" \
|
||||
--argjson prompts_multiplier "$NUM_PROMPTS_MULTIPLIER" \
|
||||
--argjson min_gain_pct "$TPS_MIN_GAIN_PCT" \
|
||||
--argjson plateau_patience "$PLATEAU_PATIENCE" \
|
||||
--argjson warmup_max_requests "$BENCH_WARMUP_MAX_REQUESTS" \
|
||||
--argjson ttft_slo_ms "${TTFT_SLO_MS:-4000}" \
|
||||
--argjson enable_ttft_slo_stop "${ENABLE_TTFT_SLO_STOP:-1}" \
|
||||
'{
|
||||
experiment: $experiment,
|
||||
engine: $engine,
|
||||
run_id: $run_id,
|
||||
model: $model,
|
||||
hardware: $hardware,
|
||||
matrix: $matrix,
|
||||
dataset: $dataset,
|
||||
tokenize_prompt: $tokenize_prompt,
|
||||
random_range_ratio: $random_range_ratio,
|
||||
search: {
|
||||
start_concurrency: $search_start,
|
||||
max_concurrency: $search_cap,
|
||||
multiplier: $search_multiplier,
|
||||
num_prompts_multiplier: $prompts_multiplier,
|
||||
min_tps_gain_pct: $min_gain_pct,
|
||||
plateau_patience: $plateau_patience,
|
||||
warmup_max_requests: $warmup_max_requests,
|
||||
ttft_slo_ms: $ttft_slo_ms,
|
||||
enable_ttft_slo_stop: $enable_ttft_slo_stop
|
||||
}
|
||||
}' > "${ADAPTIVE_RUN_ROOT}/run_manifest.json"
|
||||
"$PYTHON" - \
|
||||
"$EXPERIMENT_NAME" \
|
||||
"$ENGINE" \
|
||||
"$RUN_ID" \
|
||||
"$MODEL_PATH" \
|
||||
"$HARDWARE" \
|
||||
"$MATRIX_FILE" \
|
||||
"$BENCH_DATASET_NAME" \
|
||||
"$tokenize_prompt_json" \
|
||||
"$RANDOM_RANGE_RATIO" \
|
||||
"$SEARCH_START_CONCURRENCY" \
|
||||
"$SEARCH_MAX_CONCURRENCY" \
|
||||
"$SEARCH_MULTIPLIER" \
|
||||
"$NUM_PROMPTS_MULTIPLIER" \
|
||||
"$TPS_MIN_GAIN_PCT" \
|
||||
"$PLATEAU_PATIENCE" \
|
||||
"$BENCH_WARMUP_MAX_REQUESTS" \
|
||||
"${TTFT_SLO_MS:-4000}" \
|
||||
"${ENABLE_TTFT_SLO_STOP:-1}" \
|
||||
> "${ADAPTIVE_RUN_ROOT}/run_manifest.json" <<'PY'
|
||||
import json, sys
|
||||
|
||||
(experiment, engine, run_id, model, hardware, matrix, dataset,
|
||||
tokenize_prompt, random_range_ratio, search_start, search_cap,
|
||||
search_multiplier, prompts_multiplier, min_gain_pct, plateau_patience,
|
||||
warmup_max_requests, ttft_slo_ms, enable_ttft_slo_stop) = sys.argv[1:19]
|
||||
|
||||
print(json.dumps({
|
||||
"experiment": experiment,
|
||||
"engine": engine,
|
||||
"run_id": run_id,
|
||||
"model": model,
|
||||
"hardware": hardware,
|
||||
"matrix": matrix,
|
||||
"dataset": dataset,
|
||||
"tokenize_prompt": tokenize_prompt == "true",
|
||||
"random_range_ratio": float(random_range_ratio),
|
||||
"search": {
|
||||
"start_concurrency": int(search_start),
|
||||
"max_concurrency": int(search_cap),
|
||||
"multiplier": int(search_multiplier),
|
||||
"num_prompts_multiplier": int(prompts_multiplier),
|
||||
"min_tps_gain_pct": float(min_gain_pct),
|
||||
"plateau_patience": int(plateau_patience),
|
||||
"warmup_max_requests": int(warmup_max_requests),
|
||||
"ttft_slo_ms": float(ttft_slo_ms),
|
||||
"enable_ttft_slo_stop": int(enable_ttft_slo_stop),
|
||||
},
|
||||
}, indent=2, ensure_ascii=False))
|
||||
PY
|
||||
|
||||
ADAPTIVE_ACTIVE_TP=""
|
||||
ADAPTIVE_ACTIVE_DP=""
|
||||
|
||||
@ -176,6 +176,10 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict], meta
|
||||
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
||||
|
||||
for s in scenarios:
|
||||
if "metrics" not in s:
|
||||
# Manually-recorded skipped/failed scenario without metrics;
|
||||
# listed separately below instead of crashing the report.
|
||||
continue
|
||||
cfg = s["config"]
|
||||
m = s["metrics"]
|
||||
slo = s.get("slo_status", {}).get("overall", "")
|
||||
@ -196,6 +200,15 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict], meta
|
||||
f.write("\n")
|
||||
f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n")
|
||||
|
||||
skipped = [s for s in scenarios if "metrics" not in s]
|
||||
if skipped:
|
||||
f.write("## Skipped or failed scenarios\n\n")
|
||||
f.write("| Scenario | Status | Note |\n")
|
||||
f.write("|---|---|---|\n")
|
||||
for s in skipped:
|
||||
f.write(f"| {s['name']} | {s.get('status', '')} | {s.get('note', '')} |\n")
|
||||
f.write("\n")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user