diff --git a/.gitignore b/.gitignore index 2d047ad..9d6284d 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,8 @@ experiments/*/results/*/*/raw_outputs/ experiments/*/results/*/*/logs/ experiments/*/results/*/*.tsv experiments/*/results/*/*/*.tsv +# 平台子目录布局（experiments///results/...）下的 raw_outputs +raw_outputs/ # 运行时目录（pid 文件、缓存、临时文件） experiments/*/runtime/ diff --git a/experiments/dsv4_p800_256k_4k_probe/README.md b/experiments/p800/dsv4_p800_256k_4k_probe/README.md similarity index 86% rename from experiments/dsv4_p800_256k_4k_probe/README.md rename to experiments/p800/dsv4_p800_256k_4k_probe/README.md index ac8c6b6..1292a67 100644 --- a/experiments/dsv4_p800_256k_4k_probe/README.md +++ b/experiments/p800/dsv4_p800_256k_4k_probe/README.md @@ -16,14 +16,14 @@ without OOM or context-length errors. PLATFORM=kunlun_p800 \ SKIP_MANAGE_SERVER=1 \ PORT=30000 \ -bash experiments/dsv4_p800_256k_4k_probe/run_bench.sh +bash experiments/p800/dsv4_p800_256k_4k_probe/run_bench.sh ``` ### Standalone (starts its own Docker container) ```bash PLATFORM=kunlun_p800 \ -bash experiments/dsv4_p800_256k_4k_probe/run_bench.sh +bash experiments/p800/dsv4_p800_256k_4k_probe/run_bench.sh ``` This will start a fresh `sglang-dsv4-flash` container with the baseline @@ -34,7 +34,7 @@ To use the tuned config with EAGLE speculative decoding instead: ```bash SERVER_MODE=w8a8_int8 PLATFORM=kunlun_p800 \ -bash experiments/dsv4_p800_256k_4k_probe/run_bench.sh +bash experiments/p800/dsv4_p800_256k_4k_probe/run_bench.sh ``` ## Configuration @@ -46,7 +46,7 @@ Edit `config.env` or override via environment variables: PORT=30001 \ MODEL_PATH=/data1/models/DeepSeek-V4-Flash-INT8 \ PLATFORM=kunlun_p800 \ -bash experiments/dsv4_p800_256k_4k_probe/run_bench.sh +bash experiments/p800/dsv4_p800_256k_4k_probe/run_bench.sh ``` ## Files diff --git a/experiments/dsv4_p800_256k_4k_probe/config.env b/experiments/p800/dsv4_p800_256k_4k_probe/config.env similarity index 100% rename from experiments/dsv4_p800_256k_4k_probe/config.env rename to experiments/p800/dsv4_p800_256k_4k_probe/config.env diff --git a/experiments/dsv4_p800_256k_4k_probe/parse_results.py b/experiments/p800/dsv4_p800_256k_4k_probe/parse_results.py similarity index 100% rename from experiments/dsv4_p800_256k_4k_probe/parse_results.py rename to experiments/p800/dsv4_p800_256k_4k_probe/parse_results.py diff --git a/experiments/p800/dsv4_p800_256k_4k_probe/results/20260708-144246/results.json b/experiments/p800/dsv4_p800_256k_4k_probe/results/20260708-144246/results.json new file mode 100644 index 0000000..f94e4d8 --- /dev/null +++ b/experiments/p800/dsv4_p800_256k_4k_probe/results/20260708-144246/results.json @@ -0,0 +1,31 @@ +{ + "metadata": { + "experiment": "dsv4_p800_256k_4k_probe", + "run_id": "20260708-144246", + "timestamp": "2026-07-08T14:42:46+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_256k_4k_probe/run_bench.sh", + "env": "", + "git_commit": "89a999d", + "git_dirty": "dirty", + "description": "Kunlun P800 256k-input 4k-output OOM probe for DeepSeek-V4-Flash-INT8" + }, + "config": { + "tp": 8, + "xpu_visible_devices": "0,1,2,3,4,5,6,7", + "context_length": 270000, + "server_mode": "w8a8_int8_baseline", + "server_start_script": "experiments/dsv4_p800_256k_4k_probe/start_server.sh", + "extra_launch_args": "--context-length 270000 --max-running-requests 4", + "scenarios": [ + "2 262144 4096" + ], + "num_prompts": 4 + }, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/p800/dsv4_p800_256k_4k_probe/results/20260708-145540/results.json b/experiments/p800/dsv4_p800_256k_4k_probe/results/20260708-145540/results.json new file mode 100644 index 0000000..1855e9f --- /dev/null +++ b/experiments/p800/dsv4_p800_256k_4k_probe/results/20260708-145540/results.json @@ -0,0 +1,31 @@ +{ + "metadata": { + "experiment": "dsv4_p800_256k_4k_probe", + "run_id": "20260708-145540", + "timestamp": "2026-07-08T14:55:40+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_256k_4k_probe/run_bench.sh", + "env": "", + "git_commit": "6ec62ff", + "git_dirty": "dirty", + "description": "Kunlun P800 256k-input 4k-output OOM probe for DeepSeek-V4-Flash-INT8" + }, + "config": { + "tp": 8, + "xpu_visible_devices": "0,1,2,3,4,5,6,7", + "context_length": 270000, + "server_mode": "w8a8_int8_baseline", + "server_start_script": "experiments/dsv4_p800_256k_4k_probe/start_server.sh", + "extra_launch_args": "--context-length 270000 --max-running-requests 2 --mem-fraction-static 0.85", + "scenarios": [ + "2 262144 4096" + ], + "num_prompts": 4 + }, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_p800_256k_4k_probe/run_bench.sh b/experiments/p800/dsv4_p800_256k_4k_probe/run_bench.sh similarity index 95% rename from experiments/dsv4_p800_256k_4k_probe/run_bench.sh rename to experiments/p800/dsv4_p800_256k_4k_probe/run_bench.sh index fdf10e5..adfe559 100755 --- a/experiments/dsv4_p800_256k_4k_probe/run_bench.sh +++ b/experiments/p800/dsv4_p800_256k_4k_probe/run_bench.sh @@ -5,13 +5,13 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/server_docker.sh" +source "${SCRIPT_DIR}/../../../scripts/common/server_docker.sh" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/bench_client_docker.sh" +source "${SCRIPT_DIR}/../../../scripts/common/bench_client_docker.sh" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" diff --git a/experiments/dsv4_p800_256k_4k_probe/start_server.sh b/experiments/p800/dsv4_p800_256k_4k_probe/start_server.sh similarity index 90% rename from experiments/dsv4_p800_256k_4k_probe/start_server.sh rename to experiments/p800/dsv4_p800_256k_4k_probe/start_server.sh index c0894ed..9e58b8d 100755 --- a/experiments/dsv4_p800_256k_4k_probe/start_server.sh +++ b/experiments/p800/dsv4_p800_256k_4k_probe/start_server.sh @@ -5,7 +5,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/server_docker.sh" +source "${SCRIPT_DIR}/../../../scripts/common/server_docker.sh" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/README.md b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/README.md new file mode 100644 index 0000000..3572ffa --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/README.md @@ -0,0 +1,74 @@ +# DSV4 P800 SGLang TP×DP Matrix + +Kunlun P800 XPU + SGLang-XPU + `DeepSeek-V4-Flash-INT8` TP×DP matrix benchmark, +modeled after `experiments/h20/dsv4_h20_sglang_tp_dp_matrix`. + +## What it does + +Runs each parallel configuration with its own server instance and sweeps the +ISL/OSL matrix in `matrix.json`: + +- TP=2, DP=4 → 2 XPUs per replica, 4 replicas +- TP=4, DP=2 → 4 XPUs per replica, 2 replicas +- TP=8, DP=1 → 8 XPUs, no data parallelism (production configuration) + +Differences from the H20 version: + +- Server runs in the P800 vendor image with the proven INT8 launch args + (NSA attention backend, `w8a8_int8` quantization); + the benchmark client runs inside the same container via `docker exec`. +- ISL is capped at 131072 and concurrency ranges are lower, because P800 + INT8 sustains at most ~131k input tokens and far less concurrency. +- XPU memory sampling uses `xpu-smi` instead of `nvidia-smi`. + +`--dp-size` support on this image was probed empirically (2026-07-16, +`results/smoke_tpdp`): TP=4/DP=2 starts and serves correctly (replicas land +on disjoint XPU groups), while TP=2/DP=4 fails during model loading with a +weight-loading OOM. Root cause: expert weights are sharded only across +the TP group (this matrix passes no `--ep-size`, so experts fall back to +TP sharding; the image also asserts `ep_size <= tp_size`, so EP can never +exceed TP anyway) and DP replicas do not share expert shards. The INT8 +checkpoint holds ~264 GiB of routed-expert weights out of ~274 GiB total, +so each rank carries roughly `274/TP` GiB: +TP=8/DP=1 ≈ 34 GiB, TP=4/DP=2 ≈ 68 GiB, TP=2/DP=4 ≈ 137 GiB — more than +the 96 GiB an XPU has, hence the OOM. TP=4 is therefore the +minimum viable TP for this model on P800. Configs that cannot start are +recorded as `SKIPPED_SERVICE_START_FAILED` and the run continues with the +next config. + +## Quick Start + +```bash +# Full matrix (server per config -> warmup -> scenarios -> parse -> compare) +bash experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh + +# Dry run: print server args and scenario plan only +DRY_RUN=1 bash experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh + +# Adaptive concurrency search (find saturation point per shape) +bash experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_adaptive_concurrency.sh +``` + +Results land in `results//` (fixed matrix) or +`adaptive_results//` (adaptive search). + +## Files + +| File | Purpose | +|---|---| +| `config.env` | Experiment-level configuration (model, port, TP×DP configs, server settings) | +| `matrix.json` | ISL/OSL matrix with Y/P/N marks and per-ISL concurrency ranges | +| `generate_scenarios.py` | Expands matrix.json into a scenario TSV | +| `start_sglang_docker.sh` | Start the P800 SGLang Docker container for a TP×DP config | +| `start_sglang_dp.sh` | Alias for `start_sglang_docker.sh` (H20 naming parity) | +| `run_bench.sh` | Orchestrator: server per config → warmup → scenarios → parse → compare | +| `compare.py` | Cross-config comparison report (`comparison.md`) | +| `adaptive_config.env` | Adaptive concurrency search settings | +| `run_adaptive_concurrency.sh` | Adaptive saturation search (shared `adaptive_bench_lib.sh`) | +| `run_adaptive_concurrency_add16.sh` | Same search starting at concurrency 16, step +16 | + +## Platform + +This experiment targets `platforms/kunlun_p800.env`. The container uses the +dedicated name `sglang-dsv4-flash-tpdp` so it never interferes with the +`sglang-dsv4-flash` container used by the other P800 experiments. diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_config.env b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_config.env new file mode 100644 index 0000000..fd9e362 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_config.env @@ -0,0 +1,57 @@ +# Adaptive concurrency search settings. +# +# For each fixed (TP, DP, ISL, OSL), probe: +# C = start, start * multiplier, ... up to max +# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for +# PLATEAU_PATIENCE consecutive points. + +SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" +# The server-side max-running-requests cap has been removed; 32 is kept as +# the search upper bound to stay comparable with earlier runs. +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-32}" +SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}" +NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" + +# A gain below 2% is treated as throughput saturation. Two consecutive +# low-gain points prevent one noisy measurement from stopping the search. +TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" +PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" + +# TTFT SLO early-stop settings. +# When ttft_p95_ms exceeds TTFT_SLO_MS, stop searching the current (ISL, OSL) +# shape and move on to the next scenario. +TTFT_SLO_MS="${TTFT_SLO_MS:-4000}" +ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}" + +# Keep the same random workload semantics as the fixed matrix baseline. +# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times +# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to +# random-ids to use generated token IDs without a ShareGPT seed dataset. +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed +# ISL/OSL points; lower values intentionally benchmark a length distribution. +RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" +# NOTE: the P800 image's sglang.bench_serving does not support +# --warmup-requests, so per-point warmup requests are not sent; the value is +# still validated by adaptive_bench_lib.sh and kept for parity with H20. +BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}" + +# Reject a point if the completed request count or actual token lengths do not +# match the requested workload. +INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}" +OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}" + +MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}" +SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Optional space-separated filters, useful for smoke tests: +# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128" +TP_LIST="${TP_LIST:-}" +ISL_LIST="${ISL_LIST:-}" +OSL_LIST="${OSL_LIST:-}" + +DRY_RUN="${DRY_RUN:-0}" +# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/adaptive_points.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/adaptive_points.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/adaptive_shapes.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/adaptive_shapes.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/run_manifest.json b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/run_manifest.json new file mode 100644 index 0000000..39954a6 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/run_manifest.json @@ -0,0 +1,22 @@ +{ + "experiment": "dsv4_p800_sglang_tp_dp_matrix", + "engine": "sglang", + "run_id": "adaptive_20260716-064715", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "hardware": "8x Kunlun P800 XPU", + "matrix": "/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/matrix.json", + "dataset": "random", + "tokenize_prompt": false, + "random_range_ratio": 1.0, + "search": { + "start_concurrency": 16, + "max_concurrency": 32, + "multiplier": 2, + "num_prompts_multiplier": 5, + "min_tps_gain_pct": 2.0, + "plateau_patience": 2, + "warmup_max_requests": 0, + "ttft_slo_ms": 4000.0, + "enable_ttft_slo_stop": 1 + } +} diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/shapes.tsv b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/shapes.tsv new file mode 100644 index 0000000..f993a46 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/shapes.tsv @@ -0,0 +1,25 @@ +mark input_len output_len +Y 1024 128 +Y 1024 256 +Y 1024 512 +Y 1024 1024 +Y 1024 2048 +Y 1024 4096 +Y 4096 128 +Y 4096 256 +Y 4096 512 +Y 4096 1024 +Y 4096 2048 +Y 4096 4096 +Y 16384 128 +Y 16384 256 +Y 16384 512 +Y 16384 1024 +Y 16384 2048 +Y 65536 128 +Y 65536 256 +Y 65536 512 +Y 65536 1024 +Y 131072 128 +Y 131072 256 +Y 131072 512 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/tp2_dp4/server_cmd.txt b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/tp2_dp4/server_cmd.txt new file mode 100644 index 0000000..cddc8dc --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/tp2_dp4/server_cmd.txt @@ -0,0 +1 @@ +--host 0.0.0.0 --port 30014 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static 0.8 --tensor-parallel-size 2 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 140000 --dp-size 4 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/tp4_dp2/server_cmd.txt b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/tp4_dp2/server_cmd.txt new file mode 100644 index 0000000..b42f5f5 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/tp4_dp2/server_cmd.txt @@ -0,0 +1 @@ +--host 0.0.0.0 --port 30014 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static 0.8 --tensor-parallel-size 4 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 140000 --dp-size 2 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/compare.py b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/compare.py new file mode 100644 index 0000000..0d68bd3 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/compare.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Cross TP×DP configuration comparison for dsv4_p800_sglang_tp_dp_matrix. + +Usage: + python3 compare.py --run-root results/ [--output comparison.md] +""" +import argparse +import json +import re +from collections import defaultdict +from pathlib import Path + + +def load_result(result_root: Path) -> dict: + path = result_root / "results.json" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, + ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str: + ttft_ok = ttft_p95_ms < ttft_limit_ms + tpot_ok = tpot_mean_ms < tpot_limit_ms + if ttft_ok and tpot_ok: + return "PASS" + if ttft_ok or tpot_ok: + return "PARTIAL" + return "FAIL" + + +def gpu_memory_str(gpu: dict | None) -> str: + if not gpu: + return "-" + peak = gpu.get("peak_used_mb", 0) + total = gpu.get("memory_total_mb", 0) + if total: + return f"{peak:.0f}/{total:.0f} ({100*peak/total:.1f}%)" + return f"{peak:.0f}" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) + parser.add_argument("--ttft-limit", type=float, default=3000.0) + parser.add_argument("--tpot-limit", type=float, default=50.0) + args = parser.parse_args() + + # Discover configurations: tp*_dp* directories. + configs = [] + for subdir in sorted(args.run_root.iterdir()): + if not subdir.is_dir(): + continue + name = subdir.name + if not (name.startswith("tp") and "_dp" in name): + continue + results_json = subdir / "results.json" + if not results_json.exists(): + continue + configs.append((name, load_result(subdir))) + + if not configs: + print(f"No tp*_dp* results found under {args.run_root}") + return + + model = configs[0][1].get("metadata", {}).get("model", "unknown") + hardware = configs[0][1].get("metadata", {}).get("hardware", "unknown") + + # Group by scenario name. + by_scenario: dict[str, dict[str, dict]] = defaultdict(dict) + skipped: dict[str, dict[str, str]] = defaultdict(dict) + for label, data in configs: + for s in data.get("scenarios", []): + key = s["name"] + if s.get("status") == "skipped_oom" or "metrics" not in s: + skipped[key][label] = s.get("note") or s.get("status", "skipped") + else: + by_scenario[key][label] = s + + with open(args.output, "w", encoding="utf-8") as f: + f.write(f"# SGLang TP×DP matrix comparison ({hardware})\n\n") + f.write("## Summary\n\n") + f.write(f"- Model: `{model}`\n") + f.write(f"- Hardware: {hardware}\n") + f.write("- Backend: SGLang (Docker)\n") + f.write("- Benchmark client: `sglang.bench_serving`\n") + f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n") + + # Configuration overview. + f.write("### Configurations\n\n") + f.write("| Config | TP | DP | XPUs/replica | Notes |\n") + f.write("|---|---:|---:|---:|---|\n") + for label, data in configs: + cfg = data.get("config", {}) + tp = cfg.get("tp", "?") + dp = cfg.get("dp", "?") + f.write(f"| {label} | {tp} | {dp} | {tp} | server args recorded per ISL in results.json |\n") + f.write("\n") + + # Side-by-side table. + f.write("## Side-by-side results\n\n") + headers = [ + "Scenario", "ISL", "OSL", "Config", "Conc", "Req/s", "OutTok/s", + "TTFT P95(ms)", "TTFT P99(ms)", "TPOT Mean(ms)", "TPOT P95(ms)", + "TPOT P99(ms)", "E2E P99(ms)", "Peak GPU mem", "SLO" + ] + f.write("| " + " | ".join(headers) + " |\n") + f.write("|" + "|".join(["---"] * len(headers)) + "|\n") + + for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, re.findall(r"\d+", x)))): + _, isl, osl = re.findall(r"\d+", scenario_name) + for label, data in configs: + s = by_scenario[scenario_name].get(label) + if s is None: + if scenario_name in skipped and label in skipped[scenario_name]: + note = skipped[scenario_name][label] + f.write(f"| {scenario_name} | {isl} | {osl} | {label} | - | - | - | - | - | - | - | - | - | - | {note} |\n") + continue + cfg = s["config"] + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + gpu = m.get("gpu_memory") + f.write( + f"| {scenario_name} | {isl} | {osl} | {label} | {cfg['concurrency']} | " + f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['p99']:.2f} | {gpu_memory_str(gpu)} | {status} |\n" + ) + + # Best throughput per ISL/OSL. + f.write("\n## Best throughput per (ISL, OSL)\n\n") + f.write("| ISL | OSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |\n") + f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n") + best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {} + for scenario_name, backends in by_scenario.items(): + _, isl, osl = re.findall(r"\d+", scenario_name) + isl_i, osl_i = int(isl), int(osl) + for label, s in backends.items(): + m = s["metrics"] + out_tok = m["output_token_throughput"] + if (isl_i, osl_i) not in best_by_shape or out_tok > best_by_shape[(isl_i, osl_i)][0]: + best_by_shape[(isl_i, osl_i)] = (out_tok, label, s) + for (isl_i, osl_i), (out_tok, label, s) in sorted(best_by_shape.items()): + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + f.write( + f"| {isl_i} | {osl_i} | {label} | {s['config']['concurrency']} | " + f"{out_tok:.2f} | {m['ttft_ms']['p95']:.2f} | {m['tpot_ms']['mean']:.2f} | {status} |\n" + ) + + f.write("\n## Notes\n\n") + f.write("- SLO check uses TTFT P95 and TPOT mean.\n") + f.write("- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.\n") + f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n") + f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n") + + print(f"Wrote comparison to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/config.env b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/config.env new file mode 100644 index 0000000..45797d8 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/config.env @@ -0,0 +1,85 @@ +# TP×DP matrix experiment for DeepSeek-V4-Flash-INT8 on Kunlun P800 (8 XPUs) +# using SGLang-XPU. Tests SGLang with three parallel configurations: +# TP=2, DP=4 -> 2 XPUs per replica, 4 replicas +# TP=4, DP=2 -> 4 XPUs per replica, 2 replicas +# TP=8, DP=1 -> 8 XPUs, no data parallelism + +EXPERIMENT="dsv4_p800_sglang_tp_dp_matrix" +MODEL_NAME="DeepSeek-V4-Flash-INT8" +MODEL_PATH="${MODEL_PATH:-/data1/models/DeepSeek-V4-Flash-INT8}" +SERVED_MODEL_NAME="deepseek-v4-flash-int8" + +SGLANG_PORT="${SGLANG_PORT:-30014}" + +# Dedicated container name so this experiment never touches the +# `sglang-dsv4-flash` container used by the other P800 experiments. +CONTAINER_NAME="sglang-dsv4-flash-tpdp" + +# Python interpreter inside the P800 SGLang container (used to run the +# benchmark client via `docker exec`). +CONTAINER_PYTHON="${CONTAINER_PYTHON:-/root/miniconda/envs/python310_torch25_cuda/bin/python}" + +export XPU_VISIBLE_DEVICES="${XPU_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" + +# Runtime working directory for logs, pid files, and tmp. Defaults to a local +# directory under this experiment so the benchmark is self-contained. +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" + +# Parallel configurations to test. Format: "TP DP" +# Note: TP=2/DP=4 fails at startup with a weight-loading OOM. Root cause: +# expert weights are sharded only across the TP group (no --ep-size is +# passed, so experts fall back to TP sharding), and DP replicas do not +# share expert shards. The INT8 model holds ~264 GiB of routed-expert +# weights, so TP=2 needs ~137 GiB per XPU (>96 GiB). TP=2/DP=4 +# is kept in the default list so the matrix records the failure and moves on. +# Override with a space-separated list of comma pairs, e.g. +# PARALLEL_CONFIGS_STR="8,1" to run only TP=8/DP=1. +if [[ -n "${PARALLEL_CONFIGS_STR:-}" ]]; then + declare -a PARALLEL_CONFIGS=() + for pair in $PARALLEL_CONFIGS_STR; do + PARALLEL_CONFIGS+=("${pair//,/ }") + done +else + declare -a PARALLEL_CONFIGS=( + "2 4" + "4 2" + "8 1" + ) +fi + +# SGLang server settings. P800 INT8 supports at most ~131k input tokens, so +# the context length is capped well below the H20 1M setting. +CONTEXT_LENGTH="${CONTEXT_LENGTH:-140000}" +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.8}" + +DOCKER_IMAGE="${DOCKER_IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202}" + +# Random dataset mounted into the container at /workspace/dummy_sharegpt.json. +# DATASET_PATH is the host copy (used by the adaptive search for sanity +# checks); the benchmark client always references CONTAINER_DATASET_PATH. +PATCH_ROOT="${PATCH_ROOT:-/data1/yy/sskj/platforms/patches/kunlun_p800}" +DATASET_PATH="${DATASET_PATH:-${PATCH_ROOT}/dummy_sharegpt.json}" +CONTAINER_DATASET_PATH="${CONTAINER_DATASET_PATH:-/workspace/dummy_sharegpt.json}" + +# Matrix and concurrency rules are defined in matrix.json by default. +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" + +# Sampling density for concurrency (kept for parity with the H20 experiment; +# generate_scenarios.py currently samples the low/high endpoints only). +export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}" + +# Per-scenario timeout to avoid hangs (seconds). +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" + +# XPU memory sampling interval (seconds). +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Dry-run mode: if 1, only log the server args and scenario plan without +# starting any server or sending requests. +DRY_RUN="${DRY_RUN:-0}" + +# Per-config scenario limit for quick smoke tests. 0 = run all generated +# scenarios. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/generate_scenarios.py b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/generate_scenarios.py new file mode 100644 index 0000000..2bfb152 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/generate_scenarios.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Generate the scenario list for the TP×DP matrix experiment. + +Reads matrix.json and prints TSV lines: + + mark input_len output_len concurrency num_prompts + +mark is one of Y/P/N. The caller (run_bench.sh) decides how to treat each. +""" +import argparse +import json +import math +import os +from pathlib import Path + + +def sample_concurrency(low: int, high: int, target: int) -> list[int]: + """Return only the low and high concurrency values in [low, high]. + + For the TP×DP matrix we only need the two endpoints of the concurrency + range (e.g. 1 and 128 for ISL=1024). The `target` argument is kept for + API compatibility but is ignored. + """ + assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}" + if low == high: + return [low] + return [low, high] + + +def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]: + with open(matrix_path, "r", encoding="utf-8") as f: + data = json.load(f) + + matrix = data["matrix"] + concurrency_cfg = data["concurrency"] + + scenarios = [] + for isl_str in sorted(matrix.keys(), key=int): + osl_map = matrix[isl_str] + low = concurrency_cfg[isl_str]["low"] + high = concurrency_cfg[isl_str]["high"] + concurrencies = sample_concurrency(low, high, target_samples) + + for osl_str in sorted(osl_map.keys(), key=int): + mark = osl_map[osl_str] + if mode == "Y" and mark != "Y": + continue + if mode == "Y+P" and mark not in ("Y", "P"): + continue + # mode == "all" keeps everything, including N. + + for conc in concurrencies: + scenarios.append( + { + "mark": mark, + "input_len": int(isl_str), + "output_len": int(osl_str), + "concurrency": conc, + "num_prompts": conc * 5, + } + ) + + return scenarios + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--matrix", type=Path, default=Path("matrix.json")) + parser.add_argument("--mode", choices=["Y", "Y+P", "all"], default=None, + help="Scenario selection mode. Defaults to matrix.mode.") + parser.add_argument("--target-samples", type=int, default=0, + help="Target number of concurrency samples. 0 = heuristic (6-8).") + args = parser.parse_args() + + with open(args.matrix, "r", encoding="utf-8") as f: + data = json.load(f) + + mode = args.mode if args.mode else data.get("mode", "Y+P") + + target_samples = args.target_samples + if target_samples <= 0: + env_samples = os.getenv("CONCURRENCY_SAMPLES", "0") + try: + target_samples = int(env_samples) + except ValueError: + target_samples = 0 + if target_samples <= 0: + target_samples = 7 + + scenarios = generate_scenarios(args.matrix, mode, target_samples) + + print("mark\tinput_len\toutput_len\tconcurrency\tnum_prompts") + for s in scenarios: + print(f"{s['mark']}\t{s['input_len']}\t{s['output_len']}\t{s['concurrency']}\t{s['num_prompts']}") + + +if __name__ == "__main__": + main() diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/matrix.json b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/matrix.json new file mode 100644 index 0000000..a26bb51 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/matrix.json @@ -0,0 +1,54 @@ +{ + "comment": "ISL/OSL matrix for dsv4_p800_sglang_tp_dp_matrix. Y=must test, P=optional (record skipped on failure), N=skip.", + "mode": "Y", + "notes": "P800 INT8 sustains at most ~131k input tokens, so ISL stops at 131072 and concurrency ranges are much lower than the H20 matrix.", + "matrix": { + "1024": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "4096": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "16384": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, + "65536": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "P", + "4096": "N" + }, + "131072": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "P", + "2048": "N", + "4096": "N" + } + }, + "concurrency": { + "1024": { "low": 1, "high": 32 }, + "4096": { "low": 1, "high": 16 }, + "16384": { "low": 1, "high": 8 }, + "65536": { "low": 1, "high": 4 }, + "131072": { "low": 1, "high": 2 } + } +} diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tp8/comparison.md b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tp8/comparison.md new file mode 100644 index 0000000..a0970e3 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tp8/comparison.md @@ -0,0 +1,34 @@ +# SGLang TP×DP matrix comparison (8x Kunlun P800 XPU) + +## Summary + +- Model: `/data1/models/DeepSeek-V4-Flash-INT8` +- Hardware: 8x Kunlun P800 XPU +- Backend: SGLang (Docker) +- Benchmark client: `sglang.bench_serving` +- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms + +### Configurations + +| Config | TP | DP | XPUs/replica | Notes | +|---|---:|---:|---:|---| +| tp8_dp1 | 8 | 1 | 8 | server args recorded per ISL in results.json | + +## Side-by-side results + +| Scenario | ISL | OSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| c1_i1024_o128 | 1024 | 128 | tp8_dp1 | 1 | 0.28 | 35.50 | 0.00 | 251.79 | 26.41 | 0.00 | 26.67 | 3637.43 | 92332/98304 (93.9%) | PASS | + +## Best throughput per (ISL, OSL) + +| ISL | OSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO | +|---:|---:|---|---:|---:|---:|---:|---:| +| 1024 | 128 | tp8_dp1 | 1 | 35.50 | 0.00 | 26.41 | PASS | + +## Notes + +- SLO check uses TTFT P95 and TPOT mean. +- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out. +- `Peak GPU mem` shows peak used / total MB and utilization percentage. +- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run. diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tp8/tp8_dp1/report.md b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tp8/tp8_dp1/report.md new file mode 100644 index 0000000..5b1f134 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tp8/tp8_dp1/report.md @@ -0,0 +1,67 @@ +# 8x Kunlun P800 XPU SGLANG Benchmark Report + +- Result root: `results/smoke_tp8/tp8_dp1` +- Model: `/data1/models/DeepSeek-V4-Flash-INT8` +- Backend: SGLANG +- Benchmark client: `sglang.bench_serving --backend sglang` + +## Results + +| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Peak GPU mem | SLO | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c1_i1024_o128 | main | 1 | 1024 | 128 | 18.03 | 5 | 0.28 | 284.02 | 35.50 | 319.52 | 249.58 | 0.00 | 251.79 | 26.41 | 0.00 | 26.67 | 3603.09 | 0.00 | 3637.43 | 92332/98304 MiB (93.9%) | ✅ | + +SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail. + +## Skipped or failed scenarios + +| Scenario | Status | Note | +|---|---|---| +| c32_i1024_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i1024_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c32_i1024_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i1024_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c32_i1024_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i1024_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c32_i1024_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i1024_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c32_i1024_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i1024_o4096 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c32_i1024_o4096 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i4096_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c16_i4096_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i4096_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c16_i4096_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i4096_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c16_i4096_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i4096_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c16_i4096_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i4096_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c16_i4096_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i4096_o4096 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c16_i4096_o4096 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i16384_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c8_i16384_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i16384_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c8_i16384_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i16384_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c8_i16384_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i16384_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c8_i16384_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i16384_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c8_i16384_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i65536_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c4_i65536_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i65536_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c4_i65536_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i65536_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c4_i65536_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i65536_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c4_i65536_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i131072_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c2_i131072_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i131072_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c2_i131072_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i131072_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c2_i131072_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | + diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tp8/tp8_dp1/results.json b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tp8/tp8_dp1/results.json new file mode 100644 index 0000000..f6b0abf --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tp8/tp8_dp1/results.json @@ -0,0 +1,702 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang_tp_dp_matrix_tp8_dp1", + "run_id": "smoke_tp8", + "timestamp": "2026-07-16T05:45:26+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh", + "env": "iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202", + "git_commit": "f4f4093", + "git_dirty": "dirty", + "description": "P800 SGLang TP×DP matrix for DeepSeek-V4-Flash-INT8" + }, + "config": { + "tp": 8, + "dp": 1, + "xpu_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "sglang", + "engine": "sglang-xpu", + "server_start_script": "experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_docker.sh", + "server_args": "--host 0.0.0.0 --port 30014 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 140000 --max-running-requests 32 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 140000" + }, + "scenarios": [ + { + "name": "c1_i1024_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 18.0267706339946, + "request_throughput": 0.2773652642238138, + "input_token_throughput": 284.02203056518533, + "output_token_throughput": 35.50275382064817, + "total_token_throughput": 319.52478438583347, + "total_input_tokens": 5120, + "total_output_tokens": 640, + "e2e_ms": { + "mean": 3603.0893054325134, + "p50": 3616.7099910089746, + "p90": 3632.4909622315317, + "p95": 0.0, + "p99": 3637.4272649595514 + }, + "ttft_ms": { + "mean": 249.58492522127926, + "p50": 250.24159904569387, + "p90": 0.0, + "p95": 0.0, + "p99": 251.78673897404224 + }, + "tpot_ms": { + "mean": 26.405546300875862, + "p50": 26.52918022881284, + "p90": 0.0, + "p95": 0.0, + "p99": 26.670257044684757 + }, + "itl_ms": { + "mean": 26.405476773429047, + "p50": 24.089168990030885, + "p90": 0.0, + "p95": 47.44513131445274, + "p99": 53.76050190534443 + }, + "gpu_memory": { + "peak_used_mb": 92332.0, + "avg_used_mb": 92332.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 98304.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "results/smoke_tp8/tp8_dp1/raw_outputs/sglang_main_1_1024_128.jsonl" + }, + { + "name": "c32_i1024_o128", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i1024_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c32_i1024_o256", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i1024_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c32_i1024_o512", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i1024_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c32_i1024_o1024", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i1024_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c32_i1024_o2048", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i1024_o4096", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c32_i1024_o4096", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i4096_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c16_i4096_o128", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i4096_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c16_i4096_o256", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i4096_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c16_i4096_o512", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i4096_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c16_i4096_o1024", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i4096_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c16_i4096_o2048", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i4096_o4096", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c16_i4096_o4096", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i16384_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c8_i16384_o128", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 40 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i16384_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c8_i16384_o256", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 40 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i16384_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c8_i16384_o512", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 40 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i16384_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c8_i16384_o1024", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 40 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i16384_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c8_i16384_o2048", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 40 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i65536_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c4_i65536_o128", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i65536_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c4_i65536_o256", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i65536_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c4_i65536_o512", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i65536_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c4_i65536_o1024", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i131072_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c2_i131072_o128", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 131072, + "output_len": 128, + "dataset": "random", + "num_prompts": 10 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i131072_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c2_i131072_o256", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 10 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i131072_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c2_i131072_o512", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 10 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + } + ] +} \ No newline at end of file diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tp8/tp8_dp1/scenarios.tsv b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tp8/tp8_dp1/scenarios.tsv new file mode 100644 index 0000000..49dc9e6 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tp8/tp8_dp1/scenarios.tsv @@ -0,0 +1,49 @@ +mark input_len output_len concurrency num_prompts +Y 1024 128 1 5 +Y 1024 128 32 160 +Y 1024 256 1 5 +Y 1024 256 32 160 +Y 1024 512 1 5 +Y 1024 512 32 160 +Y 1024 1024 1 5 +Y 1024 1024 32 160 +Y 1024 2048 1 5 +Y 1024 2048 32 160 +Y 1024 4096 1 5 +Y 1024 4096 32 160 +Y 4096 128 1 5 +Y 4096 128 16 80 +Y 4096 256 1 5 +Y 4096 256 16 80 +Y 4096 512 1 5 +Y 4096 512 16 80 +Y 4096 1024 1 5 +Y 4096 1024 16 80 +Y 4096 2048 1 5 +Y 4096 2048 16 80 +Y 4096 4096 1 5 +Y 4096 4096 16 80 +Y 16384 128 1 5 +Y 16384 128 8 40 +Y 16384 256 1 5 +Y 16384 256 8 40 +Y 16384 512 1 5 +Y 16384 512 8 40 +Y 16384 1024 1 5 +Y 16384 1024 8 40 +Y 16384 2048 1 5 +Y 16384 2048 8 40 +Y 65536 128 1 5 +Y 65536 128 4 20 +Y 65536 256 1 5 +Y 65536 256 4 20 +Y 65536 512 1 5 +Y 65536 512 4 20 +Y 65536 1024 1 5 +Y 65536 1024 4 20 +Y 131072 128 1 5 +Y 131072 128 2 10 +Y 131072 256 1 5 +Y 131072 256 2 10 +Y 131072 512 1 5 +Y 131072 512 2 10 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/comparison.md b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/comparison.md new file mode 100644 index 0000000..c888e44 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/comparison.md @@ -0,0 +1,36 @@ +# SGLang TP×DP matrix comparison (8x Kunlun P800 XPU) + +## Summary + +- Model: `/data1/models/DeepSeek-V4-Flash-INT8` +- Hardware: 8x Kunlun P800 XPU +- Backend: SGLang (Docker) +- Benchmark client: `sglang.bench_serving` +- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms + +### Configurations + +| Config | TP | DP | XPUs/replica | Notes | +|---|---:|---:|---:|---| +| tp2_dp4 | 2 | 4 | 2 | server args recorded per ISL in results.json | +| tp4_dp2 | 4 | 2 | 4 | server args recorded per ISL in results.json | + +## Side-by-side results + +| Scenario | ISL | OSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| c1_i1024_o128 | 1024 | 128 | tp2_dp4 | - | - | - | - | - | - | - | - | - | - | service failed to start | +| c1_i1024_o128 | 1024 | 128 | tp4_dp2 | 1 | 0.23 | 29.80 | 0.00 | 305.76 | 31.44 | 0.00 | 31.55 | 4305.40 | 82314/98304 (83.7%) | PASS | + +## Best throughput per (ISL, OSL) + +| ISL | OSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO | +|---:|---:|---|---:|---:|---:|---:|---:| +| 1024 | 128 | tp4_dp2 | 1 | 29.80 | 0.00 | 31.44 | PASS | + +## Notes + +- SLO check uses TTFT P95 and TPOT mean. +- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out. +- `Peak GPU mem` shows peak used / total MB and utilization percentage. +- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run. diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp2_dp4/results.json b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp2_dp4/results.json new file mode 100644 index 0000000..4dcf0fb --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp2_dp4/results.json @@ -0,0 +1,653 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang_tp_dp_matrix_tp2_dp4", + "run_id": "smoke_tpdp", + "timestamp": "2026-07-16T06:00:00+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh", + "env": "iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202", + "git_commit": "f4f4093", + "git_dirty": "dirty", + "description": "P800 SGLang TP×DP matrix for DeepSeek-V4-Flash-INT8" + }, + "config": { + "tp": 2, + "dp": 4, + "xpu_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "sglang", + "engine": "sglang-xpu", + "server_start_script": "experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_docker.sh", + "server_args": "--host 0.0.0.0 --port 30014 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 140000 --max-running-requests 32 --tensor-parallel-size 2 --ep-size 2 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 140000 --dp-size 4" + }, + "scenarios": [ + { + "name": "c1_i1024_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c32_i1024_o128", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i1024_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c32_i1024_o256", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i1024_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c32_i1024_o512", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i1024_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c32_i1024_o1024", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i1024_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c32_i1024_o2048", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i1024_o4096", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c32_i1024_o4096", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i4096_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c16_i4096_o128", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i4096_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c16_i4096_o256", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i4096_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c16_i4096_o512", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i4096_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c16_i4096_o1024", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i4096_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c16_i4096_o2048", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i4096_o4096", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c16_i4096_o4096", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i16384_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c8_i16384_o128", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 40 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i16384_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c8_i16384_o256", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 40 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i16384_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c8_i16384_o512", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 40 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i16384_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c8_i16384_o1024", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 40 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i16384_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c8_i16384_o2048", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 40 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i65536_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c4_i65536_o128", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i65536_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c4_i65536_o256", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i65536_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c4_i65536_o512", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i65536_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c4_i65536_o1024", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i131072_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c2_i131072_o128", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 131072, + "output_len": 128, + "dataset": "random", + "num_prompts": 10 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i131072_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c2_i131072_o256", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 10 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c1_i131072_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + }, + { + "name": "c2_i131072_o512", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 10 + }, + "status": "SKIPPED_SERVICE_START_FAILED", + "note": "service failed to start" + } + ] +} \ No newline at end of file diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp2_dp4/scenarios.tsv b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp2_dp4/scenarios.tsv new file mode 100644 index 0000000..49dc9e6 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp2_dp4/scenarios.tsv @@ -0,0 +1,49 @@ +mark input_len output_len concurrency num_prompts +Y 1024 128 1 5 +Y 1024 128 32 160 +Y 1024 256 1 5 +Y 1024 256 32 160 +Y 1024 512 1 5 +Y 1024 512 32 160 +Y 1024 1024 1 5 +Y 1024 1024 32 160 +Y 1024 2048 1 5 +Y 1024 2048 32 160 +Y 1024 4096 1 5 +Y 1024 4096 32 160 +Y 4096 128 1 5 +Y 4096 128 16 80 +Y 4096 256 1 5 +Y 4096 256 16 80 +Y 4096 512 1 5 +Y 4096 512 16 80 +Y 4096 1024 1 5 +Y 4096 1024 16 80 +Y 4096 2048 1 5 +Y 4096 2048 16 80 +Y 4096 4096 1 5 +Y 4096 4096 16 80 +Y 16384 128 1 5 +Y 16384 128 8 40 +Y 16384 256 1 5 +Y 16384 256 8 40 +Y 16384 512 1 5 +Y 16384 512 8 40 +Y 16384 1024 1 5 +Y 16384 1024 8 40 +Y 16384 2048 1 5 +Y 16384 2048 8 40 +Y 65536 128 1 5 +Y 65536 128 4 20 +Y 65536 256 1 5 +Y 65536 256 4 20 +Y 65536 512 1 5 +Y 65536 512 4 20 +Y 65536 1024 1 5 +Y 65536 1024 4 20 +Y 131072 128 1 5 +Y 131072 128 2 10 +Y 131072 256 1 5 +Y 131072 256 2 10 +Y 131072 512 1 5 +Y 131072 512 2 10 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp4_dp2/report.md b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp4_dp2/report.md new file mode 100644 index 0000000..9b0d3f5 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp4_dp2/report.md @@ -0,0 +1,67 @@ +# 8x Kunlun P800 XPU SGLANG Benchmark Report + +- Result root: `/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp4_dp2` +- Model: `/data1/models/DeepSeek-V4-Flash-INT8` +- Backend: SGLANG +- Benchmark client: `sglang.bench_serving --backend sglang` + +## Results + +| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Peak GPU mem | SLO | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c1_i1024_o128 | main | 1 | 1024 | 128 | 21.48 | 5 | 0.23 | 238.40 | 29.80 | 268.20 | 299.56 | 0.00 | 305.76 | 31.44 | 0.00 | 31.55 | 4292.17 | 0.00 | 4305.40 | 82314/98304 MiB (83.7%) | ✅ | + +SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail. + +## Skipped or failed scenarios + +| Scenario | Status | Note | +|---|---|---| +| c32_i1024_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i1024_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c32_i1024_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i1024_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c32_i1024_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i1024_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c32_i1024_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i1024_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c32_i1024_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i1024_o4096 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c32_i1024_o4096 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i4096_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c16_i4096_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i4096_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c16_i4096_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i4096_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c16_i4096_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i4096_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c16_i4096_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i4096_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c16_i4096_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i4096_o4096 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c16_i4096_o4096 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i16384_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c8_i16384_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i16384_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c8_i16384_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i16384_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c8_i16384_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i16384_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c8_i16384_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i16384_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c8_i16384_o2048 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i65536_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c4_i65536_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i65536_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c4_i65536_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i65536_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c4_i65536_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i65536_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c4_i65536_o1024 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i131072_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c2_i131072_o128 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i131072_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c2_i131072_o256 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c1_i131072_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | +| c2_i131072_o512 | SKIPPED_GRID_LIMIT | GRID_LIMIT reached | + diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp4_dp2/results.json b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp4_dp2/results.json new file mode 100644 index 0000000..525ea81 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp4_dp2/results.json @@ -0,0 +1,702 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang_tp_dp_matrix_tp4_dp2", + "run_id": "smoke_tpdp", + "timestamp": "2026-07-16T06:00:58+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh", + "env": "iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202", + "git_commit": "f4f4093", + "git_dirty": "dirty", + "description": "P800 SGLang TP×DP matrix for DeepSeek-V4-Flash-INT8" + }, + "config": { + "tp": 4, + "dp": 2, + "xpu_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "sglang", + "engine": "sglang-xpu", + "server_start_script": "experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_docker.sh", + "server_args": "--host 0.0.0.0 --port 30014 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 140000 --max-running-requests 32 --tensor-parallel-size 4 --ep-size 4 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 140000 --dp-size 2" + }, + "scenarios": [ + { + "name": "c1_i1024_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "metrics": { + "success": 5, + "failed": 0, + "duration_s": 21.476578783942387, + "request_throughput": 0.23281175508914861, + "input_token_throughput": 238.39923721128818, + "output_token_throughput": 29.799904651411023, + "total_token_throughput": 268.1991418626992, + "total_input_tokens": 5120, + "total_output_tokens": 640, + "e2e_ms": { + "mean": 4292.167780222371, + "p50": 4291.483911918476, + "p90": 4302.627834840678, + "p95": 0.0, + "p99": 4305.40240991395 + }, + "ttft_ms": { + "mean": 299.5579913724214, + "p50": 298.9764349767938, + "p90": 0.0, + "p95": 0.0, + "p99": 305.76277466956526 + }, + "tpot_ms": { + "mean": 31.43787235314921, + "p50": 31.393611976898857, + "p90": 0.0, + "p95": 0.0, + "p99": 31.546158453038945 + }, + "itl_ms": { + "mean": 31.43780216387671, + "p50": 29.1481880703941, + "p90": 0.0, + "p95": 55.60536030679939, + "p99": 62.21151448087765 + }, + "gpu_memory": { + "peak_used_mb": 82314.0, + "avg_used_mb": 82118.0, + "peak_utilization_pct": 100.0, + "memory_total_mb": 98304.0 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp4_dp2/raw_outputs/sglang_main_1_1024_128.jsonl" + }, + { + "name": "c32_i1024_o128", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i1024_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c32_i1024_o256", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i1024_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c32_i1024_o512", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i1024_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c32_i1024_o1024", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i1024_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c32_i1024_o2048", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i1024_o4096", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c32_i1024_o4096", + "config": { + "phase": "main", + "concurrency": 32, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 160 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i4096_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c16_i4096_o128", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i4096_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c16_i4096_o256", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i4096_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c16_i4096_o512", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i4096_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c16_i4096_o1024", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i4096_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c16_i4096_o2048", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i4096_o4096", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c16_i4096_o4096", + "config": { + "phase": "main", + "concurrency": 16, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 80 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i16384_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c8_i16384_o128", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 40 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i16384_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c8_i16384_o256", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 40 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i16384_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c8_i16384_o512", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 40 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i16384_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c8_i16384_o1024", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 40 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i16384_o2048", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c8_i16384_o2048", + "config": { + "phase": "main", + "concurrency": 8, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 40 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i65536_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c4_i65536_o128", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i65536_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c4_i65536_o256", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i65536_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c4_i65536_o512", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i65536_o1024", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c4_i65536_o1024", + "config": { + "phase": "main", + "concurrency": 4, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i131072_o128", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 128, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c2_i131072_o128", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 131072, + "output_len": 128, + "dataset": "random", + "num_prompts": 10 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i131072_o256", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c2_i131072_o256", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 10 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c1_i131072_o512", + "config": { + "phase": "main", + "concurrency": 1, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 5 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + }, + { + "name": "c2_i131072_o512", + "config": { + "phase": "main", + "concurrency": 2, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 10 + }, + "status": "SKIPPED_GRID_LIMIT", + "note": "GRID_LIMIT reached" + } + ] +} \ No newline at end of file diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp4_dp2/scenarios.tsv b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp4_dp2/scenarios.tsv new file mode 100644 index 0000000..49dc9e6 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/results/smoke_tpdp/tp4_dp2/scenarios.tsv @@ -0,0 +1,49 @@ +mark input_len output_len concurrency num_prompts +Y 1024 128 1 5 +Y 1024 128 32 160 +Y 1024 256 1 5 +Y 1024 256 32 160 +Y 1024 512 1 5 +Y 1024 512 32 160 +Y 1024 1024 1 5 +Y 1024 1024 32 160 +Y 1024 2048 1 5 +Y 1024 2048 32 160 +Y 1024 4096 1 5 +Y 1024 4096 32 160 +Y 4096 128 1 5 +Y 4096 128 16 80 +Y 4096 256 1 5 +Y 4096 256 16 80 +Y 4096 512 1 5 +Y 4096 512 16 80 +Y 4096 1024 1 5 +Y 4096 1024 16 80 +Y 4096 2048 1 5 +Y 4096 2048 16 80 +Y 4096 4096 1 5 +Y 4096 4096 16 80 +Y 16384 128 1 5 +Y 16384 128 8 40 +Y 16384 256 1 5 +Y 16384 256 8 40 +Y 16384 512 1 5 +Y 16384 512 8 40 +Y 16384 1024 1 5 +Y 16384 1024 8 40 +Y 16384 2048 1 5 +Y 16384 2048 8 40 +Y 65536 128 1 5 +Y 65536 128 4 20 +Y 65536 256 1 5 +Y 65536 256 4 20 +Y 65536 512 1 5 +Y 65536 512 4 20 +Y 65536 1024 1 5 +Y 65536 1024 4 20 +Y 131072 128 1 5 +Y 131072 128 2 10 +Y 131072 256 1 5 +Y 131072 256 2 10 +Y 131072 512 1 5 +Y 131072 512 2 10 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_adaptive_concurrency.sh new file mode 100755 index 0000000..16b6056 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_adaptive_concurrency.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape +# on Kunlun P800 (DeepSeek-V4-Flash-INT8). +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="sglang" +ENGINE_PORT="$SGLANG_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +PYTHON="${PYTHON:-$(command -v python3)}" + +# The shared library's GPU monitor uses nvidia-smi; override it with an +# xpu-smi sampler that emits the same CSV shape. +adaptive_start_gpu_monitor() { + local csv_path="$1" + mkdir -p "$(dirname "$csv_path")" + ( + echo "timestamp, index, memory.used [MiB], memory.total [MiB], utilization.gpu [%]" + while true; do + xpu-smi 2>/dev/null | awk -F'|' -v ts="$(date '+%Y/%m/%d %H:%M:%S')" ' + /^\|[[:space:]]*[0-9]+[[:space:]]/ { + l=$0; sub(/^\|[[:space:]]*/, "", l); sub(/[[:space:]].*$/, "", l); idx=l + } + /MiB/ && $4 ~ /%/ { + split($3, a, "/"); u=a[1]; t=a[2] + gsub(/[^0-9.]/, "", u); gsub(/[^0-9.]/, "", t) + r=$4; gsub(/[^0-9.]/, "", r) + printf "%s, %s, %s MiB, %s MiB, %s %%\n", ts, idx, u, t, r + }' + sleep "$GPU_MEM_SAMPLE_INTERVAL_S" + done + ) >> "$csv_path" 2>/dev/null & + echo $! +} + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + # The server only ever runs inside this experiment's dedicated container, + # so removing the container is a complete and safe stop. + log "stopping container ${CONTAINER_NAME} tp=${tp} dp=${dp}" + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + # Keep in sync with the launch args in start_sglang_docker.sh. + local tp="$1" + local dp="$2" + local args="--host 0.0.0.0 --port ${ENGINE_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH}" + if (( dp > 1 )); then + args="${args} --dp-size ${dp}" + fi + printf '%s' "$args" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + log "starting sglang server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_sglang_docker.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + if ! engine_is_healthy; then + log "ERROR: sglang health check failed tp=${tp} dp=${dp}" + return 1 + fi + ACTIVE_ENGINE_SERVER_LOG="" + log "sglang server healthy tp=${tp} dp=${dp} log=docker logs ${CONTAINER_NAME}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + if grep -Eiq "$pattern" "$detail_log" 2>/dev/null; then + return 0 + fi + docker logs --tail 2000 "$CONTAINER_NAME" 2>/dev/null | grep -Eiq "$pattern" +} + +engine_run_bench() { + local isl="$1" + local osl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local container_output="/tmp/bench_outputs/adaptive_$(basename "$output_file")" + if [[ "$output_file" == "/dev/null" ]]; then + container_output="/dev/null" + fi + + local -a bench_args=( + --backend sglang + --host 127.0.0.1 + --port "$ENGINE_PORT" + --model "$MODEL_PATH" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$osl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --output-file "$container_output" + --output-details + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$CONTAINER_DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + docker exec "$CONTAINER_NAME" mkdir -p /tmp/bench_outputs 2>/dev/null || true + docker exec "$CONTAINER_NAME" \ + env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \ + "$CONTAINER_PYTHON" -m sglang.bench_serving "${bench_args[@]}" || return $? + + if [[ "$output_file" != "/dev/null" ]]; then + docker cp "$CONTAINER_NAME:${container_output}" "$output_file" || return 1 + fi +} + +export -f engine_run_bench +export ENGINE_PORT CONTAINER_NAME CONTAINER_PYTHON MODEL_PATH RESULT_BASE +export BENCH_DATASET_NAME CONTAINER_DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +adaptive_main "$@" diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh new file mode 100755 index 0000000..0e6f32f --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape +# on Kunlun P800 (DeepSeek-V4-Flash-INT8). +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="sglang" +ENGINE_PORT="$SGLANG_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +PYTHON="${PYTHON:-$(command -v python3)}" + +# The shared library's GPU monitor uses nvidia-smi; override it with an +# xpu-smi sampler that emits the same CSV shape. +adaptive_start_gpu_monitor() { + local csv_path="$1" + mkdir -p "$(dirname "$csv_path")" + ( + echo "timestamp, index, memory.used [MiB], memory.total [MiB], utilization.gpu [%]" + while true; do + xpu-smi 2>/dev/null | awk -F'|' -v ts="$(date '+%Y/%m/%d %H:%M:%S')" ' + /^\|[[:space:]]*[0-9]+[[:space:]]/ { + l=$0; sub(/^\|[[:space:]]*/, "", l); sub(/[[:space:]].*$/, "", l); idx=l + } + /MiB/ && $4 ~ /%/ { + split($3, a, "/"); u=a[1]; t=a[2] + gsub(/[^0-9.]/, "", u); gsub(/[^0-9.]/, "", t) + r=$4; gsub(/[^0-9.]/, "", r) + printf "%s, %s, %s MiB, %s MiB, %s %%\n", ts, idx, u, t, r + }' + sleep "$GPU_MEM_SAMPLE_INTERVAL_S" + done + ) >> "$csv_path" 2>/dev/null & + echo $! +} + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + # The server only ever runs inside this experiment's dedicated container, + # so removing the container is a complete and safe stop. + log "stopping container ${CONTAINER_NAME} tp=${tp} dp=${dp}" + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + # Keep in sync with the launch args in start_sglang_docker.sh. + local tp="$1" + local dp="$2" + local args="--host 0.0.0.0 --port ${ENGINE_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH}" + if (( dp > 1 )); then + args="${args} --dp-size ${dp}" + fi + printf '%s' "$args" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + log "starting sglang server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_sglang_docker.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + if ! engine_is_healthy; then + log "ERROR: sglang health check failed tp=${tp} dp=${dp}" + return 1 + fi + ACTIVE_ENGINE_SERVER_LOG="" + log "sglang server healthy tp=${tp} dp=${dp} log=docker logs ${CONTAINER_NAME}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + if grep -Eiq "$pattern" "$detail_log" 2>/dev/null; then + return 0 + fi + docker logs --tail 2000 "$CONTAINER_NAME" 2>/dev/null | grep -Eiq "$pattern" +} + +engine_run_bench() { + local isl="$1" + local osl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local container_output="/tmp/bench_outputs/adaptive_$(basename "$output_file")" + if [[ "$output_file" == "/dev/null" ]]; then + container_output="/dev/null" + fi + + local -a bench_args=( + --backend sglang + --host 127.0.0.1 + --port "$ENGINE_PORT" + --model "$MODEL_PATH" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$osl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --output-file "$container_output" + --output-details + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$CONTAINER_DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + docker exec "$CONTAINER_NAME" mkdir -p /tmp/bench_outputs 2>/dev/null || true + docker exec "$CONTAINER_NAME" \ + env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \ + "$CONTAINER_PYTHON" -m sglang.bench_serving "${bench_args[@]}" || return $? + + if [[ "$output_file" != "/dev/null" ]]; then + docker cp "$CONTAINER_NAME:${container_output}" "$output_file" || return 1 + fi +} + +export -f engine_run_bench +export ENGINE_PORT CONTAINER_NAME CONTAINER_PYTHON MODEL_PATH RESULT_BASE +export BENCH_DATASET_NAME CONTAINER_DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +export SEARCH_START_CONCURRENCY=16 +export SEARCH_ADDEND=16 + +adaptive_main "$@" diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh new file mode 100755 index 0000000..bb643c8 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh @@ -0,0 +1,555 @@ +#!/usr/bin/env bash +# TP×DP matrix benchmark for DeepSeek-V4-Flash-INT8 on Kunlun P800 SGLang (Docker). +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_BASE="${SCRIPT_DIR}/results" +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" +DRY_RUN="${DRY_RUN:-0}" +GRID_LIMIT="${GRID_LIMIT:-0}" + +PYTHON="${PYTHON:-$(command -v python3)}" + +# Export variables used inside functions that are called via bash -c subshells. +export CONTAINER_NAME CONTAINER_PYTHON MODEL_PATH SGLANG_PORT CONTAINER_DATASET_PATH + +log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" +mkdir -p "$log_dir_global" +log_init "${log_dir_global}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}" +log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE} dry_run=${DRY_RUN} grid_limit=${GRID_LIMIT}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +is_server_healthy() { + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1 +} + +container_running() { + docker inspect "$CONTAINER_NAME" >/dev/null 2>&1 +} + +stop_server() { + local tp="$1" + local dp="$2" + + # The server only ever runs inside this experiment's dedicated container, + # so removing the container is a complete and safe stop. + if container_running; then + log "stopping container ${CONTAINER_NAME} (tp=${tp}, dp=${dp})" + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + else + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + fi + sleep 2 +} + +build_server_args() { + # Keep in sync with the launch args in start_sglang_docker.sh. + local tp="$1" + local dp="$2" + + local args="--host 0.0.0.0 --port ${SGLANG_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH}" + if [[ "$dp" -gt 1 ]]; then + args="${args} --dp-size ${dp}" + fi + printf '%s' "$args" +} + +start_server() { + local tp="$1" + local dp="$2" + + log "starting sglang server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_sglang_docker.sh" "$tp" "$dp" \ + >> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 + + if ! is_server_healthy; then + log "error: sglang server tp=${tp} dp=${dp} failed health check on port ${SGLANG_PORT}" + return 1 + fi + log "sglang server tp=${tp} dp=${dp} is healthy on port ${SGLANG_PORT}" +} + +restart_server() { + local tp="$1" + local dp="$2" + log "restarting sglang server tp=${tp} dp=${dp} after non-OOM failure" + stop_server "$tp" "$dp" + sleep 10 + start_server "$tp" "$dp" +} + +run_bench_serving() { + # Run sglang.bench_serving inside the server container (the P800 image + # carries the only working client environment for this fork). + docker exec "$CONTAINER_NAME" \ + env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \ + "$CONTAINER_PYTHON" -m sglang.bench_serving "$@" +} +export -f run_bench_serving + +run_warmup() { + local input_len="$1" + local output_len="$2" + + log "warming up (input=${input_len}, output=${output_len}, num=1)" + bash -c ' + run_bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "'"$SGLANG_PORT"'" \ + --model "'"$MODEL_PATH"'" \ + --dataset-name random \ + --dataset-path "'"$CONTAINER_DATASET_PATH"'" \ + --random-input-len "'"$input_len"'" \ + --random-output-len "'"$output_len"'" \ + --random-range-ratio 1.0 \ + --num-prompts 1 \ + --max-concurrency 1 \ + --request-rate 10000 \ + --output-file /dev/null \ + --output-details \ + >> "'"${log_dir_global}/warmup.log"'" 2>&1 + ' || log "WARNING: warmup failed; continuing with config" + log "warmup completed" +} + +scenario_already_completed() { + local output_file="$1" + local expected="$2" + [[ -s "$output_file" ]] || return 1 + local completed + completed="$("$PYTHON" -c " +import json, sys +path = sys.argv[1] +try: + with open(path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line: + data = json.loads(line) + print(data.get('completed', 0)) + break +except Exception: + print(0) +" "$output_file")" + [[ "${completed:-0}" -ge "$expected" ]] +} + +scenario_already_processed() { + local result_root="$1" + local scenario_name="$2" + local json_path="${result_root}/results.json" + [[ -f "$json_path" ]] || return 1 + "$PYTHON" -c " +import json, sys +path, name = sys.argv[1], sys.argv[2] +try: + with open(path, 'r', encoding='utf-8') as f: + data = json.load(f) + for s in data.get('scenarios', []): + if s.get('name') == name: + if s.get('status') or s.get('metrics', {}).get('success', 0) > 0: + sys.exit(0) +except Exception: + pass +sys.exit(1) +" "$json_path" "$scenario_name" +} + +detect_oom() { + local detail_log="$1" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + if grep -Eiq "$pattern" "$detail_log" 2>/dev/null; then + return 0 + fi + docker logs --tail 2000 "$CONTAINER_NAME" 2>/dev/null | grep -Eiq "$pattern" +} + +start_gpu_monitor() { + # Sample XPU memory/utilization with xpu-smi and emit nvidia-smi-style CSV + # rows so scripts/common/parse_backend.py can read them unchanged. + local csv_path="$1" + mkdir -p "$(dirname "$csv_path")" + ( + echo "timestamp, index, memory.used [MiB], memory.total [MiB], utilization.gpu [%]" + while true; do + xpu-smi 2>/dev/null | awk -F'|' -v ts="$(date '+%Y/%m/%d %H:%M:%S')" ' + /^\|[[:space:]]*[0-9]+[[:space:]]/ { + l=$0; sub(/^\|[[:space:]]*/, "", l); sub(/[[:space:]].*$/, "", l); idx=l + } + /MiB/ && $4 ~ /%/ { + split($3, a, "/"); u=a[1]; t=a[2] + gsub(/[^0-9.]/, "", u); gsub(/[^0-9.]/, "", t) + r=$4; gsub(/[^0-9.]/, "", r) + printf "%s, %s, %s MiB, %s MiB, %s %%\n", ts, idx, u, t, r + }' + sleep "$GPU_MEM_SAMPLE_INTERVAL_S" + done + ) >> "$csv_path" 2>/dev/null & + echo $! +} + +stop_gpu_monitor() { + local pid="$1" + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi +} + +scenario_config_json() { + "$PYTHON" -c " +import json, sys +print(json.dumps({ + 'phase': 'main', + 'concurrency': int(sys.argv[1]), + 'input_len': int(sys.argv[2]), + 'output_len': int(sys.argv[3]), + 'dataset': 'random', + 'num_prompts': int(sys.argv[4]), +})) +" "$1" "$2" "$3" "$4" +} + +append_scenario_record() { + local result_root="$1" + local json_path="$result_root/results.json" + shift + local scenario_json + scenario_json="$("$PYTHON" -c " +import json, sys +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v +print(json.dumps(d, ensure_ascii=False)) +" "$@")" + PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json" +} + +record_skipped_csv() { + local csv_path="$1" + shift + # Args: key=value + local row + row="$("$PYTHON" -c " +import csv, json, sys, io +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v +buf = io.StringIO() +writer = csv.DictWriter(buf, fieldnames=['engine','tp','dp','mark','isl','osl','concurrency','status','reason','detail_log'], extrasaction='ignore') +writer.writerow(d) +print(buf.getvalue().strip()) +" "$@")" + echo "$row" >> "$csv_path" +} + +skip_remaining_scenarios() { + local result_root="$1" + local scenario_tsv="$2" + local start_index="$3" + local status="$4" + local reason="$5" + local tp="$6" + local dp="$7" + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + + local i=0 + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl osl conc num; do + if (( i < start_index )); then + i=$((i + 1)) + continue + fi + i=$((i + 1)) + local sname="c${conc}_i${isl}_o${osl}" + if scenario_already_processed "$result_root" "$sname"; then + continue + fi + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(scenario_config_json "$conc" "$isl" "$osl" "$num")" \ + "status=\"${status}\"" \ + "note=\"${reason}\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=${status}" "reason=${reason}" + done +} + +# --------------------------------------------------------------------------- +# Per-configuration runner +# --------------------------------------------------------------------------- + +run_parallel_config() { + local tp="$1" + local dp="$2" + local config_label="tp${tp}_dp${dp}" + local result_root="${RESULT_BASE}/${RUN_ID}/${config_label}" + local raw_dir="${result_root}/raw_outputs" + local gpu_log_dir="${result_root}/gpu_logs" + local phase_log_dir="${result_root}/logs" + mkdir -p "$raw_dir" "$gpu_log_dir" "$phase_log_dir" + + log "===== ${config_label} START =====" + + # Generate scenario list for this config. + local scenario_tsv="${result_root}/scenarios.tsv" + "$PYTHON" "${SCRIPT_DIR}/generate_scenarios.py" \ + --matrix "$MATRIX_FILE" \ + --mode "$MATRIX_MODE" \ + > "$scenario_tsv" + local total_scenarios + total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)" + log "generated ${total_scenarios} scenarios for ${config_label}" + + # Write metadata. + ensure_result_root "$result_root" + write_metadata_json \ + "${result_root}/results.json" \ + "${EXPERIMENT_NAME}_${config_label}" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "sglang" \ + "sglang-xpu" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/p800/${EXPERIMENT_NAME}/run_bench.sh" \ + "$DOCKER_IMAGE" \ + "P800 SGLang TP×DP matrix for DeepSeek-V4-Flash-INT8" + + local server_args_str + server_args_str="$(build_server_args "$tp" "$dp")" + "$PYTHON" - "$tp" "$dp" "$server_args_str" "${result_root}/results.json" <<'PY' +import json, os, sys + +tp, dp, server_args, path = int(sys.argv[1]), int(sys.argv[2]), sys.argv[3], sys.argv[4] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) +data["config"] = { + "tp": tp, + "dp": dp, + "xpu_visible_devices": os.environ.get("XPU_VISIBLE_DEVICES", "0,1,2,3,4,5,6,7"), + "backend": "sglang", + "engine": "sglang-xpu", + "server_start_script": "experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_docker.sh", + "server_args": server_args, +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY + + if [[ "$DRY_RUN" == "1" ]]; then + log "DRY_RUN: would start server with args: ${server_args_str}" + local line + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl osl conc num; do + log "DRY_RUN: ${config_label} scenario mark=${mark} c=${conc} i=${isl} o=${osl} n=${num}" + done + log "===== ${config_label} DONE (dry run) =====" + return 0 + fi + + # Initialize skipped_after_oom.csv for this run. + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + if [[ ! -f "$skipped_csv" ]]; then + echo "engine,tp,dp,mark,isl,osl,concurrency,status,reason,detail_log" > "$skipped_csv" + fi + + # Start server once for this TP×DP config. + if ! start_server "$tp" "$dp"; then + log "ERROR: ${config_label} failed to start; skipping all scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" 0 "SKIPPED_SERVICE_START_FAILED" "service failed to start" "$tp" "$dp" + log "===== ${config_label} DONE =====" + return 0 + fi + + # Warmup with a small prompt before the first scenario. + run_warmup 1024 128 || true + + # Read scenarios into an array so we can skip remaining entries on failure. + local -a scenarios=() + while IFS= read -r line; do + scenarios+=("$line") + done < <(tail -n +2 "$scenario_tsv") + + local i mark isl osl conc num + local output_file container_output detail_log gpu_csv sname bench_rc + for (( i = 0; i < ${#scenarios[@]}; i++ )); do + IFS=$'\t' read -r mark isl osl conc num <<< "${scenarios[$i]}" + + if [[ "$GRID_LIMIT" -gt 0 && "$i" -ge "$GRID_LIMIT" ]]; then + log "GRID_LIMIT=${GRID_LIMIT} reached; skipping remaining scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_GRID_LIMIT" "GRID_LIMIT reached" "$tp" "$dp" + break + fi + + sname="c${conc}_i${isl}_o${osl}" + output_file="${raw_dir}/sglang_main_${conc}_${isl}_${osl}.jsonl" + container_output="/tmp/bench_outputs/sglang_main_${conc}_${isl}_${osl}.jsonl" + detail_log="${phase_log_dir}/sglang_${config_label}_${sname}.log" + gpu_csv="${gpu_log_dir}/gpu_mem_${conc}_${isl}_${osl}.csv" + + if scenario_already_completed "$output_file" "$num" || scenario_already_processed "$result_root" "$sname"; then + log "skipping already-processed ${config_label} scenario: ${sname}" + continue + fi + + if ! container_running; then + log "WARNING: container ${CONTAINER_NAME} died, skipping remaining scenarios in ${config_label}" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_CONTAINER_DIED" "container died" "$tp" "$dp" + break + fi + + log "running ${config_label} scenario: mark=${mark} c=${conc} i=${isl} o=${osl} n=${num}" + + local gpu_pid + gpu_pid="$(start_gpu_monitor "$gpu_csv")" + + bench_rc=0 + docker exec "$CONTAINER_NAME" mkdir -p /tmp/bench_outputs + timeout "$SCENARIO_TIMEOUT_S" bash -c ' + run_bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "'"$SGLANG_PORT"'" \ + --model "'"$MODEL_PATH"'" \ + --dataset-name random \ + --dataset-path "'"$CONTAINER_DATASET_PATH"'" \ + --random-input-len "'"$isl"'" \ + --random-output-len "'"$osl"'" \ + --random-range-ratio 1.0 \ + --num-prompts "'"$num"'" \ + --max-concurrency "'"$conc"'" \ + --request-rate 10000 \ + --output-file "'"$container_output"'" \ + --output-details \ + > "'"$detail_log"'" 2>&1 + ' || bench_rc=$? + + stop_gpu_monitor "$gpu_pid" + + if [[ "$bench_rc" -eq 0 ]]; then + if docker cp "${CONTAINER_NAME}:${container_output}" "$output_file"; then + log "finished ${config_label} scenario: output=${output_file}" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(scenario_config_json "$conc" "$isl" "$osl" "$num")" \ + "status=\"completed\"" \ + "note=\"benchmark finished successfully\"" + continue + fi + log "ERROR: failed to copy ${container_output} from container" + bench_rc=1 + fi + + # Failure handling. + if detect_oom "$detail_log"; then + log "ERROR: ${config_label} scenario ${sname} triggered OOM; stopping config" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(scenario_config_json "$conc" "$isl" "$osl" "$num")" \ + "status=\"OOM\"" \ + "note=\"detected out-of-memory\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=OOM" "reason=detected out-of-memory" "detail_log=${detail_log}" + stop_server "$tp" "$dp" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_AFTER_OOM" "previous case OOM" "$tp" "$dp" + break + fi + + log "ERROR: ${config_label} scenario ${sname} failed (rc=${bench_rc}); see ${detail_log}" + if [[ "$mark" == "P" ]]; then + log "optional (P) scenario failed; recording as skipped and continuing" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(scenario_config_json "$conc" "$isl" "$osl" "$num")" \ + "status=\"skipped_optional\"" \ + "note=\"optional scenario failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=skipped_optional" "reason=optional scenario failed (rc=${bench_rc})" "detail_log=${detail_log}" + continue + fi + + # Mandatory scenario failed but not OOM: try to restart the server. + if restart_server "$tp" "$dp"; then + run_warmup 1024 128 || true + log "resuming ${config_label} after server restart" + continue + fi + + log "ERROR: ${config_label} server restart failed; skipping remaining scenarios" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(scenario_config_json "$conc" "$isl" "$osl" "$num")" \ + "status=\"FAILED\"" \ + "note=\"scenario failed and server restart failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=FAILED" "reason=scenario failed and server restart failed" "detail_log=${detail_log}" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_RESTART_FAILED" "server restart failed" "$tp" "$dp" + break + done + + stop_server "$tp" "$dp" + + # Parse results. + log "parsing ${config_label} results" + "$PYTHON" "${SCRIPT_DIR}/../../../scripts/common/parse_backend.py" "$result_root" --backend sglang \ + >> "${phase_log_dir}/parse.log" 2>&1 || { + log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log" + } + + log "===== ${config_label} DONE =====" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +# Cleanup any leftovers. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + stop_server "$tp" "$dp" +done + +# Run each parallel configuration. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + run_parallel_config "$tp" "$dp" +done + +# Generate cross-configuration comparison. +log "generating comparison report" +"$PYTHON" "${SCRIPT_DIR}/compare.py" \ + --run-root "${RESULT_BASE}/${RUN_ID}" \ + --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ + >> "${log_dir_global}/compare.log" 2>&1 || { + log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" + } + +log "all results saved to ${RESULT_BASE}/${RUN_ID}" diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_docker.sh b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_docker.sh new file mode 100755 index 0000000..cb704f3 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_docker.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# Start the P800 SGLang INT8 server in Docker for a given TP×DP configuration. +# Usage: start_sglang_docker.sh +# +# Based on the proven P800 INT8 launch from dsv4_p800_long_context_matrix, +# with --tensor-parallel-size/--dp-size parameterized per config. +set -Eeuo pipefail + +TP="${1:-}" +DP="${2:-}" +if [[ -z "$TP" || -z "$DP" ]]; then + echo "Usage: $0 " + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +PORT="${SGLANG_PORT:-30014}" +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" +SERVER_LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" + +log "starting P800 SGLang INT8 server (tp=${TP}, dp=${DP})" +log "model: ${MODEL_PATH}" +log "port: ${PORT}" +log "container: ${CONTAINER_NAME}" + +# Stop any existing container with the same name. +docker rm -f "$CONTAINER_NAME" 2>/dev/null || true + +# Build device args. +device_args="" +for i in 0 1 2 3 4 5 6 7; do + device_args="${device_args} --device /dev/xpu${i}:/dev/xpu${i}" +done +device_args="${device_args} --device /dev/xpuctrl:/dev/xpuctrl" + +# Environment variables required by the P800 SGLang INT8 image. +env_args=( + -e XPU_VISIBLE_DEVICES="${XPU_VISIBLE_DEVICES}" + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + -e CUDA_DEVICE_ORDER=OAM_ID + -e SGLANG_USE_TRANSFORMERS_V5_TOKENIZER=1 + -e XMLIR_FORCE_USE_XPU_GRAPH=1 + -e SGLANG_DSV4_MODE=2604 + -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + -e SGLANG_NSA_DUAL_STREAM=true + -e SGLANG_NSA_QUANT_WQ_B_WK=false + -e SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 + -e SGLANG_OPT_DEEPGEMM_HC_PRENORM=false + -e SGLANG_OPT_USE_TILELANG_MHC_PRE=1 + -e SGLANG_OPT_USE_TILELANG_MHC_POST=1 + -e SGLANG_CLEAN_REQUEST_WHEN_RETRACT=1 + -e SGLANG_SET_CPU_AFFINITY=1 + -e SGLANG_OPT_USE_KLX_TOPK_KERNEL=1 + -e XSGL_INTERTYPE_BFP16=1 + -e ENABLE_FAST_BFP16_ATTN=1 + -e XSGL_USE_DEEP_GEMM_BMM=1 + -e XSGL_XDNN_QUANT=1 + -e XSGL_FUSE_RMS_NORM_QUANT=1 + -e XSGL_TRANSPOSE_MATMUL_WEIGHT=1 + -e XINFER_QUANT_SDNN=1 + -e XSGL_USE_MOE_SIGMOID_GROUP_TOPK_NORM=1 + -e XSGL_EARLY_FIRST_TOKEN=1 + -e XSGL_ENABLE_TGEMM_FP16=1 + -e SGLANG_ENABLE_SPEC_V2=True + -e SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + -e PYTHONDONTWRITEBYTECODE=1 + -e XTORCH_OPS_LIB_DIR=/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/xtorch_ops + -e XPU_RUNTIME_LIB_DIR=/root/miniconda/envs/python310_torch25_cuda/xcudart/lib + -e BKCL_TREE_THRESHOLD=1048576 + -e CUDA_ENABLE_P2P_NO_UVA=1 + -e NCCL_IB_GID_INDEX=3 + -e IS_DSV4=1 + -e MC_CUSTOM_TOPO_JSON=/workspace/nic_priority_matrix_test.json + # INT8 specific + -e SGLANG_DSV4_FP4_EXPERTS=false + -e SGLANG_APPLY_CONFIG_BACKUP=auto + -e BKCL_ENABLE_XDR=1 + -e BKCL_RDMA_NICS=eth1,eth1,eth3,eth3,eth5,eth5,eth7,eth7 + -e BKCL_RDMA_VERBS=1 + -e XSGL_INT8_LM_HEAD=1 + -e SGLANG_P800_ALL_GATHER_FALLBACK=0 +) + +# Launch args. Same proven P800 INT8 command as +# dsv4_p800_long_context_matrix/start_sglang.sh, with TP/DP parameterized. +# Note: whether this image supports --dp-size on XPU is part of what this +# experiment probes; TP=8/DP=1 matches the production configuration. +launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${TP} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH}" + +if [[ "$DP" -gt 1 ]]; then + launch_args="${launch_args} --dp-size ${DP}" +fi + +# Base64-encode the bootstrap command to avoid host-shell quoting issues. +server_cmd=$(cat </dev/null || true +/root/miniconda/envs/python310_torch25_cuda/bin/pip install --upgrade safetensors -q +/root/miniconda/envs/python310_torch25_cuda/bin/pip install https://files.pythonhosted.org/packages/14/8b/2a1333a6455c6fad401c2285dee6f58016c55b1cb44cae3a31f8a9cc7d83/apache_tvm_ffi-0.1.0b2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl -q +/root/miniconda/envs/python310_torch25_cuda/bin/python -c "import torch; torch.float8_e8m0fnu = torch.uint8; import runpy, sys; sys.argv[0] = 'sglang.launch_server'; runpy.run_module('sglang.launch_server', run_name='__main__')" ${launch_args} +EOF +) +server_cmd_b64=$(printf '%s' "$server_cmd" | base64 -w0) + +patch_mounts=( + -v "${PATCH_ROOT}/nic_priority_matrix_test.json:/workspace/nic_priority_matrix_test.json:ro" + -v "${PATCH_ROOT}/dummy_sharegpt.json:/workspace/dummy_sharegpt.json:ro" +) + +echo "=== Starting P800 SGLang INT8 server in Docker (TP=${TP}, DP=${DP}) ===" +echo "Image: ${DOCKER_IMAGE}" +echo "Model: ${MODEL_PATH}" +echo "Container name: ${CONTAINER_NAME}" +echo "Host port: ${PORT}" +echo "Command: sglang.launch_server ${launch_args}" +echo "Log: docker logs ${CONTAINER_NAME}" + +docker run -d \ + --name "${CONTAINER_NAME}" \ + --privileged \ + --network host \ + --ipc host \ + ${device_args} \ + -v "${MODEL_PATH}:/models:ro" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + "${patch_mounts[@]}" \ + "${env_args[@]}" \ + "${DOCKER_IMAGE}" \ + bash -c "echo '${server_cmd_b64}' | base64 -d | bash" \ + >> "${SERVER_LOG}" 2>&1 + +log "container ${CONTAINER_NAME} started, waiting for health" +healthy=0 +for ((i = 1; i <= 600; i++)); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + healthy=1 + break + fi + if [[ "$(docker inspect -f '{{.State.Running}}' "${CONTAINER_NAME}" 2>/dev/null)" != "true" ]]; then + log "ERROR: container ${CONTAINER_NAME} exited during startup; last server logs:" + docker logs --tail 100 "${CONTAINER_NAME}" 2>&1 || true + exit 1 + fi + sleep 1 +done +if [[ "$healthy" == "1" ]]; then + log "container ${CONTAINER_NAME} is healthy" + exit 0 +else + log "ERROR: container ${CONTAINER_NAME} failed health check; last server logs:" + docker logs --tail 100 "${CONTAINER_NAME}" 2>&1 || true + exit 1 +fi diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_dp.sh b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_dp.sh new file mode 100755 index 0000000..8939f73 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_dp.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Start SGLang server for a given TP×DP configuration. +# Usage: start_sglang_dp.sh +# +# P800 SGLang always runs in the vendor Docker image, so this simply +# delegates to the Docker start script. Kept under the same name as the +# H20 experiment for parity. +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "${SCRIPT_DIR}/start_sglang_docker.sh" "$@" diff --git a/scripts/common/adaptive_bench_lib.sh b/scripts/common/adaptive_bench_lib.sh index fcd06f4..6e8a180 100755 --- a/scripts/common/adaptive_bench_lib.sh +++ b/scripts/common/adaptive_bench_lib.sh @@ -134,40 +134,42 @@ adaptive_append_point_from_metrics() { local gain_json gain_json="$(adaptive_json_number_or_null "$gain_pct")" - jq -c \ - --arg timestamp "$(adaptive_timestamp)" \ - --arg engine "$ENGINE" \ - --argjson tp "$tp" \ - --argjson dp "$dp" \ - --arg mark "$mark" \ - --argjson isl "$isl" \ - --argjson osl "$osl" \ - --argjson concurrency "$concurrency" \ - --argjson num_prompts "$num_prompts" \ - --argjson warmup_requests "$(adaptive_warmup_request_count "$concurrency")" \ - --argjson attempt "$attempt" \ - --argjson gain_pct "$gain_json" \ - --argjson plateau_streak "$plateau_streak" \ - --arg raw_file "$raw_file" \ - --arg detail_log "$detail_log" \ - '. + { - timestamp: $timestamp, - engine: $engine, - tp: $tp, - dp: $dp, - mark: $mark, - isl: $isl, - osl: $osl, - concurrency: $concurrency, - num_prompts: $num_prompts, - warmup_requests: $warmup_requests, - attempt: $attempt, - gain_pct: $gain_pct, - plateau_streak: $plateau_streak, - error_type: "", - raw_file: $raw_file, - detail_log: $detail_log - }' "$metrics_file" >> "$ADAPTIVE_POINTS_JSONL" + "$PYTHON" - \ + "$metrics_file" \ + "$(adaptive_timestamp)" \ + "$ENGINE" \ + "$tp" "$dp" "$mark" "$isl" "$osl" "$concurrency" "$num_prompts" \ + "$(adaptive_warmup_request_count "$concurrency")" \ + "$attempt" "$gain_json" "$plateau_streak" "$raw_file" "$detail_log" \ + >> "$ADAPTIVE_POINTS_JSONL" <<'PY' +import json, sys + +(path, timestamp, engine, tp, dp, mark, isl, osl, concurrency, + num_prompts, warmup_requests, attempt, gain_json, plateau_streak, + raw_file, detail_log) = sys.argv[1:17] + +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) +data.update({ + "timestamp": timestamp, + "engine": engine, + "tp": int(tp), + "dp": int(dp), + "mark": mark, + "isl": int(isl), + "osl": int(osl), + "concurrency": int(concurrency), + "num_prompts": int(num_prompts), + "warmup_requests": int(warmup_requests), + "attempt": int(attempt), + "gain_pct": json.loads(gain_json), + "plateau_streak": int(plateau_streak), + "error_type": "", + "raw_file": raw_file, + "detail_log": detail_log, +}) +print(json.dumps(data, separators=(",", ":"), ensure_ascii=False)) +PY } adaptive_append_failed_point() { @@ -184,44 +186,42 @@ adaptive_append_failed_point() { local raw_file="${11}" local detail_log="${12}" - jq -nc \ - --arg timestamp "$(adaptive_timestamp)" \ - --arg engine "$ENGINE" \ - --argjson tp "$tp" \ - --argjson dp "$dp" \ - --arg mark "$mark" \ - --argjson isl "$isl" \ - --argjson osl "$osl" \ - --argjson concurrency "$concurrency" \ - --argjson num_prompts "$num_prompts" \ - --argjson warmup_requests "$(adaptive_warmup_request_count "$concurrency")" \ - --argjson attempt "$attempt" \ - --arg status "$status" \ - --arg error_type "$error_type" \ - --arg raw_file "$raw_file" \ - --arg detail_log "$detail_log" \ - '{ - timestamp: $timestamp, - engine: $engine, - tp: $tp, - dp: $dp, - mark: $mark, - isl: $isl, - osl: $osl, - concurrency: $concurrency, - num_prompts: $num_prompts, - warmup_requests: $warmup_requests, - attempt: $attempt, - status: $status, - completed: 0, - failed: $num_prompts, - error_type: $error_type, - validation_errors: [], - gain_pct: null, - plateau_streak: 0, - raw_file: $raw_file, - detail_log: $detail_log - }' >> "$ADAPTIVE_POINTS_JSONL" + "$PYTHON" - \ + "$(adaptive_timestamp)" \ + "$ENGINE" \ + "$tp" "$dp" "$mark" "$isl" "$osl" "$concurrency" "$num_prompts" \ + "$(adaptive_warmup_request_count "$concurrency")" \ + "$attempt" "$status" "$error_type" "$raw_file" "$detail_log" \ + >> "$ADAPTIVE_POINTS_JSONL" <<'PY' +import json, sys + +(timestamp, engine, tp, dp, mark, isl, osl, concurrency, num_prompts, + warmup_requests, attempt, status, error_type, raw_file, + detail_log) = sys.argv[1:16] + +print(json.dumps({ + "timestamp": timestamp, + "engine": engine, + "tp": int(tp), + "dp": int(dp), + "mark": mark, + "isl": int(isl), + "osl": int(osl), + "concurrency": int(concurrency), + "num_prompts": int(num_prompts), + "warmup_requests": int(warmup_requests), + "attempt": int(attempt), + "status": status, + "completed": 0, + "failed": int(num_prompts), + "error_type": error_type, + "validation_errors": [], + "gain_pct": None, + "plateau_streak": 0, + "raw_file": raw_file, + "detail_log": detail_log, +}, separators=(",", ":"), ensure_ascii=False)) +PY } adaptive_append_shape_summary() { @@ -246,43 +246,40 @@ adaptive_append_shape_summary() { best_tps_json="$(adaptive_json_number_or_null "$best_total_tps")" last_tps_json="$(adaptive_json_number_or_null "$last_total_tps")" - jq -nc \ - --arg timestamp "$(adaptive_timestamp)" \ - --arg engine "$ENGINE" \ - --argjson tp "$tp" \ - --argjson dp "$dp" \ - --arg mark "$mark" \ - --argjson isl "$isl" \ - --argjson osl "$osl" \ - --arg status "$status" \ - --arg stop_reason "$stop_reason" \ - --argjson tested_points "$tested_points" \ - --argjson search_cap "$SEARCH_MAX_CONCURRENCY" \ - --argjson max_successful_concurrency "$max_successful_concurrency" \ - --argjson saturation_concurrency "$saturation_json" \ - --argjson stop_probe_concurrency "$stop_probe_json" \ - --argjson best_tps_concurrency "$best_c_json" \ - --argjson best_total_tps "$best_tps_json" \ - --argjson last_total_tps "$last_tps_json" \ - '{ - timestamp: $timestamp, - engine: $engine, - tp: $tp, - dp: $dp, - mark: $mark, - isl: $isl, - osl: $osl, - status: $status, - stop_reason: $stop_reason, - tested_points: $tested_points, - search_cap: $search_cap, - max_successful_concurrency: $max_successful_concurrency, - saturation_concurrency: $saturation_concurrency, - stop_probe_concurrency: $stop_probe_concurrency, - best_tps_concurrency: $best_tps_concurrency, - best_total_tps: $best_total_tps, - last_total_tps: $last_total_tps - }' >> "$ADAPTIVE_SHAPES_JSONL" + "$PYTHON" - \ + "$(adaptive_timestamp)" \ + "$ENGINE" \ + "$tp" "$dp" "$mark" "$isl" "$osl" "$status" "$stop_reason" \ + "$tested_points" "$SEARCH_MAX_CONCURRENCY" "$max_successful_concurrency" \ + "$saturation_json" "$stop_probe_json" "$best_c_json" "$best_tps_json" "$last_tps_json" \ + >> "$ADAPTIVE_SHAPES_JSONL" <<'PY' +import json, sys + +(timestamp, engine, tp, dp, mark, isl, osl, status, stop_reason, + tested_points, search_cap, max_successful_concurrency, + saturation_json, stop_probe_json, best_c_json, best_tps_json, + last_tps_json) = sys.argv[1:18] + +print(json.dumps({ + "timestamp": timestamp, + "engine": engine, + "tp": int(tp), + "dp": int(dp), + "mark": mark, + "isl": int(isl), + "osl": int(osl), + "status": status, + "stop_reason": stop_reason, + "tested_points": int(tested_points), + "search_cap": int(search_cap), + "max_successful_concurrency": int(max_successful_concurrency), + "saturation_concurrency": json.loads(saturation_json), + "stop_probe_concurrency": json.loads(stop_probe_json), + "best_tps_concurrency": json.loads(best_c_json), + "best_total_tps": json.loads(best_tps_json), + "last_total_tps": json.loads(last_tps_json), +}, separators=(",", ":"), ensure_ascii=False)) +PY } adaptive_run_point() { @@ -389,7 +386,7 @@ adaptive_run_shape() { stop_probe_concurrency="$concurrency" if (( point_rc == 0 )); then - current_tps="$(jq -r '.total_tps' "$POINT_METRICS_FILE")" + current_tps="$("$PYTHON" -c 'import json, sys; print(json.load(open(sys.argv[1]))["total_tps"])' "$POINT_METRICS_FILE")" gain_pct="" meaningful_gain=1 if [[ -n "$previous_tps" ]]; then @@ -420,7 +417,7 @@ adaptive_run_shape() { # TTFT SLO check: stop searching if TTFT P95 exceeds the SLO threshold local ttft_p95_ms="" if [[ "${ENABLE_TTFT_SLO_STOP:-1}" == "1" ]]; then - ttft_p95_ms="$(jq -r '.ttft_p95_ms' "$POINT_METRICS_FILE")" + ttft_p95_ms="$("$PYTHON" -c 'import json, sys; print(json.load(open(sys.argv[1]))["ttft_p95_ms"])' "$POINT_METRICS_FILE")" if awk -v ttft="$ttft_p95_ms" -v slo="${TTFT_SLO_MS:-4000}" 'BEGIN { exit !(ttft > slo) }'; then adaptive_append_point_from_metrics \ "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \ @@ -543,9 +540,26 @@ adaptive_shape_already_tested() { return 1 fi local count - count="$(jq -s -r --argjson tp "$tp" --argjson dp "$dp" --arg mark "$mark" --argjson isl "$isl" --argjson osl "$osl" ' - [ .[] | select(.tp == $tp and .dp == $dp and .mark == $mark and .isl == $isl and .osl == $osl) ] | length - ' "$resume_shapes_jsonl" 2>/dev/null)" + count="$("$PYTHON" - "$tp" "$dp" "$mark" "$isl" "$osl" "$resume_shapes_jsonl" 2>/dev/null <<'PY' +import json, sys + +tp, dp, mark, isl, osl = int(sys.argv[1]), int(sys.argv[2]), sys.argv[3], int(sys.argv[4]), int(sys.argv[5]) +count = 0 +with open(sys.argv[6], "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + d = json.loads(line) + except Exception: + continue + if d.get("tp") == tp and d.get("dp") == dp and d.get("mark") == mark \ + and d.get("isl") == isl and d.get("osl") == osl: + count += 1 +print(count) +PY +)" if [[ -z "$count" ]]; then return 1 fi @@ -633,47 +647,56 @@ adaptive_main() { log "search_start=${SEARCH_START_CONCURRENCY} search_cap=${SEARCH_MAX_CONCURRENCY} multiplier=${SEARCH_MULTIPLIER} min_gain_pct=${TPS_MIN_GAIN_PCT} patience=${PLATEAU_PATIENCE} prompts_multiplier=${NUM_PROMPTS_MULTIPLIER} warmup_cap=${BENCH_WARMUP_MAX_REQUESTS}" log "dataset=${BENCH_DATASET_NAME} random_range_ratio=${RANDOM_RANGE_RATIO} tokenize_prompt=${tokenize_prompt_json} matrix=${MATRIX_FILE} mode=${MATRIX_MODE}" - jq -n \ - --arg experiment "$EXPERIMENT_NAME" \ - --arg engine "$ENGINE" \ - --arg run_id "$RUN_ID" \ - --arg model "$MODEL_PATH" \ - --arg hardware "$HARDWARE" \ - --arg matrix "$MATRIX_FILE" \ - --arg dataset "$BENCH_DATASET_NAME" \ - --argjson tokenize_prompt "$tokenize_prompt_json" \ - --argjson random_range_ratio "$RANDOM_RANGE_RATIO" \ - --argjson search_start "$SEARCH_START_CONCURRENCY" \ - --argjson search_cap "$SEARCH_MAX_CONCURRENCY" \ - --argjson search_multiplier "$SEARCH_MULTIPLIER" \ - --argjson prompts_multiplier "$NUM_PROMPTS_MULTIPLIER" \ - --argjson min_gain_pct "$TPS_MIN_GAIN_PCT" \ - --argjson plateau_patience "$PLATEAU_PATIENCE" \ - --argjson warmup_max_requests "$BENCH_WARMUP_MAX_REQUESTS" \ - --argjson ttft_slo_ms "${TTFT_SLO_MS:-4000}" \ - --argjson enable_ttft_slo_stop "${ENABLE_TTFT_SLO_STOP:-1}" \ - '{ - experiment: $experiment, - engine: $engine, - run_id: $run_id, - model: $model, - hardware: $hardware, - matrix: $matrix, - dataset: $dataset, - tokenize_prompt: $tokenize_prompt, - random_range_ratio: $random_range_ratio, - search: { - start_concurrency: $search_start, - max_concurrency: $search_cap, - multiplier: $search_multiplier, - num_prompts_multiplier: $prompts_multiplier, - min_tps_gain_pct: $min_gain_pct, - plateau_patience: $plateau_patience, - warmup_max_requests: $warmup_max_requests, - ttft_slo_ms: $ttft_slo_ms, - enable_ttft_slo_stop: $enable_ttft_slo_stop - } - }' > "${ADAPTIVE_RUN_ROOT}/run_manifest.json" + "$PYTHON" - \ + "$EXPERIMENT_NAME" \ + "$ENGINE" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "$HARDWARE" \ + "$MATRIX_FILE" \ + "$BENCH_DATASET_NAME" \ + "$tokenize_prompt_json" \ + "$RANDOM_RANGE_RATIO" \ + "$SEARCH_START_CONCURRENCY" \ + "$SEARCH_MAX_CONCURRENCY" \ + "$SEARCH_MULTIPLIER" \ + "$NUM_PROMPTS_MULTIPLIER" \ + "$TPS_MIN_GAIN_PCT" \ + "$PLATEAU_PATIENCE" \ + "$BENCH_WARMUP_MAX_REQUESTS" \ + "${TTFT_SLO_MS:-4000}" \ + "${ENABLE_TTFT_SLO_STOP:-1}" \ + > "${ADAPTIVE_RUN_ROOT}/run_manifest.json" <<'PY' +import json, sys + +(experiment, engine, run_id, model, hardware, matrix, dataset, + tokenize_prompt, random_range_ratio, search_start, search_cap, + search_multiplier, prompts_multiplier, min_gain_pct, plateau_patience, + warmup_max_requests, ttft_slo_ms, enable_ttft_slo_stop) = sys.argv[1:19] + +print(json.dumps({ + "experiment": experiment, + "engine": engine, + "run_id": run_id, + "model": model, + "hardware": hardware, + "matrix": matrix, + "dataset": dataset, + "tokenize_prompt": tokenize_prompt == "true", + "random_range_ratio": float(random_range_ratio), + "search": { + "start_concurrency": int(search_start), + "max_concurrency": int(search_cap), + "multiplier": int(search_multiplier), + "num_prompts_multiplier": int(prompts_multiplier), + "min_tps_gain_pct": float(min_gain_pct), + "plateau_patience": int(plateau_patience), + "warmup_max_requests": int(warmup_max_requests), + "ttft_slo_ms": float(ttft_slo_ms), + "enable_ttft_slo_stop": int(enable_ttft_slo_stop), + }, +}, indent=2, ensure_ascii=False)) +PY ADAPTIVE_ACTIVE_TP="" ADAPTIVE_ACTIVE_DP="" diff --git a/scripts/common/parse_backend.py b/scripts/common/parse_backend.py index d269dc1..7414a9d 100755 --- a/scripts/common/parse_backend.py +++ b/scripts/common/parse_backend.py @@ -176,6 +176,10 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict], meta f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") for s in scenarios: + if "metrics" not in s: + # Manually-recorded skipped/failed scenario without metrics; + # listed separately below instead of crashing the report. + continue cfg = s["config"] m = s["metrics"] slo = s.get("slo_status", {}).get("overall", "") @@ -196,6 +200,15 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict], meta f.write("\n") f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") + skipped = [s for s in scenarios if "metrics" not in s] + if skipped: + f.write("## Skipped or failed scenarios\n\n") + f.write("| Scenario | Status | Note |\n") + f.write("|---|---|---|\n") + for s in skipped: + f.write(f"| {s['name']} | {s.get('status', '')} | {s.get('note', '')} |\n") + f.write("\n") + def main() -> None: parser = argparse.ArgumentParser()