!16 feat(p800): add Qwen3-235B-A22B SGLang TP=8 benchmark experiment
Merge pull request !16 from Zhiyi Hong/auto/main/11665927/214b0db4-1
This commit is contained in:
Quantong Qiu
Gitee
commit
d0319504e4
63
experiments/p800/qwen3_235b_p800_sglang_tp8/README.md
Normal file
63
experiments/p800/qwen3_235b_p800_sglang_tp8/README.md
Normal file
@ -0,0 +1,63 @@
|
||||
# Qwen3-235B-A22B · Kunlun P800 · SGLang TP=8
|
||||
|
||||
在 p800.2（8× 昆仑芯 P800 XPU，每卡 96GB）上用 SGLang 以 **TP=8** 部署
|
||||
`Qwen3-235B-A22B`，并用 `sglang.bench_serving` 跑 **输入 2048 / 输出 2048 / 并发 16**
|
||||
的吞吐基准，输出吞吐、TTFT、TPOT、ITL、E2E 等指标。
|
||||
|
||||
## 关键设计
|
||||
|
||||
本实验借鉴仓库里已验证可用的 **qwen3-8b TP=1** 启动命令（同一昆仑 P800 sglang 镜像），
|
||||
仅做以下适配：
|
||||
|
||||
| 项 | qwen3-8b (TP=1) | 本实验 (TP=8) |
|
||||
|---|---|---|
|
||||
| 容器 XPU | 仅 `/dev/xpu0` | `/dev/xpu0..7`（新容器） |
|
||||
| `--tp-size` | 1 | 8 |
|
||||
| `XPU_VISIBLE_DEVICES` | 0 | 0,1,2,3,4,5,6,7 |
|
||||
| `--model-path` | Qwen3-8B | Qwen3-235B-A22B |
|
||||
| `--context-length` | 4096 | 8192（8b 用 4096 在 2k+2k 下截断了 2 条输出） |
|
||||
| 其余启动参数 | — | 完全一致 |
|
||||
|
||||
不变的 Qwen3 专用配置（与 DSV4 实验不同）：
|
||||
- `--attention-backend kunlun`（非 DSV4 的 `nsa`）
|
||||
- XPU 环境变量：`XSGL_INTERTYPE_BFP16=1`、`XSGL_FAST_SWIGLU=1`、
|
||||
`XMLIR_FORCE_USE_XPU_GRAPH=1`、`XSGL_FUSE_SPLIT_NORM_ROPE_NEOX=1`
|
||||
- `--dtype float16`、`--disable-radix-cache`、`--chunked-prefill-size -1`、
|
||||
`--page-size 64`、`--mem-fraction-static 0.9`、`--cuda-graph-bs 1..16`
|
||||
- 无 DSV4 专用变量（`IS_DSV4`/`SGLANG_DSV4_MODE` 等）、无 reasoning/tool-call parser
|
||||
|
||||
显存预算：235B FP16 权重 ~470GB，TP=8 每卡 ~59GB；每卡 96GB，0.9 静态占比留
|
||||
~27GB/卡 给 KV+激活+cuda graph，2k+2k 上下文 KV 极小，充裕。
|
||||
|
||||
> 注意：框架 `scripts/common/server_docker.sh` 里的 `docker_server_start` 是
|
||||
> DeepSeek-V4 硬编码的（NSA attention、`IS_DSV4=1`、deepseekv4 parser），
|
||||
> **不能**用于 Qwen3，因此本实验自带独立的 `start_server.sh`。
|
||||
|
||||
## 文件
|
||||
|
||||
- `config.env` — 实验参数唯一来源
|
||||
- `start_server.sh` — 起 8-XPU 容器 + 容器内启动 sglang + 健康检查
|
||||
- `run_bench.sh` — 编排入口（写元数据 → 起服务 → warmup+bench → 解析）
|
||||
- `parse_results.py` — 解析 `sglang.bench_serving` 输出（复用自 dsv4_p800_sglang）
|
||||
- `results/<run_id>/` — 自动生成：`raw_outputs/*.jsonl`、`logs/`、`results.json`、`report.md`
|
||||
|
||||
## 用法
|
||||
|
||||
```bash
|
||||
cd /data1/yy/sskj/experiments/p800/qwen3_235b_p800_sglang_tp8
|
||||
|
||||
# 一键跑（起服务 + 基准 + 解析）
|
||||
bash run_bench.sh
|
||||
|
||||
# 或分步：先起服务（便于排查），再复用服务跑基准
|
||||
bash start_server.sh
|
||||
SKIP_MANAGE_SERVER=1 bash run_bench.sh
|
||||
```
|
||||
|
||||
服务启动加载 235B 权重（~470GB）+ 捕获 16 个 batch size 的 cuda graph 较慢，
|
||||
健康检查超时设为 1200s。结果在 `results/<run_id>/report.md`。
|
||||
|
||||
## 输出指标（report.md）
|
||||
|
||||
每个 scenario 给出：Success/Failed、Req/s、Input/Output/Total tok/s、
|
||||
TTFT（mean/p50/p90/p99）、TPOT（mean/p50/p90/p99）、E2E（p99）等。
|
||||
51
experiments/p800/qwen3_235b_p800_sglang_tp8/config.env
Normal file
51
experiments/p800/qwen3_235b_p800_sglang_tp8/config.env
Normal file
@ -0,0 +1,51 @@
|
||||
# Experiment: Qwen3-235B-A22B on Kunlun P800 (8x XPU), SGLang, TP=8.
|
||||
# Benchmark: input 2048 / output 2048, concurrency 16, num_prompts 160.
|
||||
#
|
||||
# Adapted from the proven qwen3-8b TP=1 launch on this same image
|
||||
# (attention-backend kunlun, XSGL_* XPU env vars, --dtype float16).
|
||||
# Only differences vs the 8b run: --tp-size 8, 8 XPU devices, model path,
|
||||
# and context-length 8192 (8b used 4096 which truncated 2 outputs at 2k+2k).
|
||||
|
||||
EXPERIMENT="${EXPERIMENT:-qwen3_235b_p800_sglang_tp8}"
|
||||
MODEL_NAME="${MODEL_NAME:-Qwen3-235B-A22B}"
|
||||
MODEL_PATH="${MODEL_PATH:-/data1/models/Qwen3-235B-A22B}"
|
||||
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-/data1/models/Qwen3-235B-A22B}"
|
||||
BACKEND="${BACKEND:-sglang}"
|
||||
ENGINE="${ENGINE:-sglang-xpu}"
|
||||
DATASET="${DATASET:-random}"
|
||||
DATASET_PATH="${DATASET_PATH:-/data1/yy/sharegpt_data/ShareGPT_V3_unfiltered_cleaned_split.json}"
|
||||
|
||||
# Serving port + container.
|
||||
# 30000 is taken by the lingering qwen3_8b_bench_tp1 container (docker-proxy),
|
||||
# so default to 30010.
|
||||
PORT="${PORT:-30010}"
|
||||
CONTAINER_NAME="${CONTAINER_NAME:-qwen3_235b_bench_tp8}"
|
||||
|
||||
# Docker image: Kunlun P800 SGLang image (same one the qwen3-8b run used).
|
||||
DOCKER_IMAGE="${DOCKER_IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202}"
|
||||
|
||||
# Hardware: 8x Kunlun P800 XPU, TP=8.
|
||||
TP="${TP:-8}"
|
||||
XPU_VISIBLE_DEVICES="${XPU_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||||
CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||||
|
||||
# SGLang launch parameters (mirrors the working qwen3-8b config, TP bumped to 8).
|
||||
CONTEXT_LENGTH="${CONTEXT_LENGTH:-8192}"
|
||||
MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-16}"
|
||||
ATTENTION_BACKEND="${ATTENTION_BACKEND:-kunlun}"
|
||||
MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.9}"
|
||||
DTYPE="${DTYPE:-float16}"
|
||||
PAGE_SIZE="${PAGE_SIZE:-64}"
|
||||
MAX_PREFILL_TOKENS="${MAX_PREFILL_TOKENS:-32768}"
|
||||
CUDA_GRAPH_MAX_BS="${CUDA_GRAPH_MAX_BS:-16}"
|
||||
CUDA_GRAPH_BS="${CUDA_GRAPH_BS:-1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}"
|
||||
|
||||
# Bench scenario: "concurrency input_len output_len num_prompts".
|
||||
SCENARIOS=(
|
||||
"16 2048 2048 160"
|
||||
)
|
||||
|
||||
WARMUP="${WARMUP:-16}"
|
||||
REQUEST_RATE="${REQUEST_RATE:-10000}"
|
||||
SEED="${SEED:-1}"
|
||||
RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1}"
|
||||
313
experiments/p800/qwen3_235b_p800_sglang_tp8/parse_results.py
Executable file
313
experiments/p800/qwen3_235b_p800_sglang_tp8/parse_results.py
Executable file
@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parse P800 SGLang benchmark outputs and produce results.json + report.md.
|
||||
|
||||
Usage:
|
||||
python3 parse_results.py <result_root>
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections import OrderedDict
|
||||
|
||||
METRIC_PATTERNS = OrderedDict(
|
||||
[
|
||||
("successful_requests", [r"Successful requests:\s+(\d+)"]),
|
||||
("benchmark_duration_s", [r"Benchmark duration \(s\):\s+([\d.]+)"]),
|
||||
("request_throughput", [r"Request throughput \(req/s\):\s+([\d.]+)"]),
|
||||
("input_token_throughput", [r"Input token throughput \(tok/s\):\s+([\d.]+)"]),
|
||||
("output_token_throughput", [r"Output token throughput \(tok/s\):\s+([\d.]+)"]),
|
||||
("total_token_throughput", [r"Total token throughput \(tok/s\):\s+([\d.]+)"]),
|
||||
("ttft_mean", [r"Mean TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("ttft_p50", [r"Median TTFT \(ms\):\s+([\d.]+)", r"P50 TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("ttft_p90", [r"P90 TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("ttft_p99", [r"P99 TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_mean", [r"Mean TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_p50", [r"Median TPOT \(ms\):\s+([\d.]+)", r"P50 TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_p90", [r"P90 TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_p99", [r"P99 TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_mean", [r"Mean E2E \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_p50", [r"Median E2E \(ms\):\s+([\d.]+)", r"P50 E2E \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_p90", [r"P90 E2E \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_p99", [r"P99 E2E \(ms\):\s+([\d.]+)"]),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def find_float(text: str, patterns: list[str]) -> float | None:
|
||||
for pat in patterns:
|
||||
m = re.search(pat, text)
|
||||
if m:
|
||||
try:
|
||||
return float(m.group(1))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def parse_summary_log(log_text: str) -> dict:
|
||||
result = {}
|
||||
for key, patterns in METRIC_PATTERNS.items():
|
||||
result[key] = find_float(log_text, patterns)
|
||||
return result
|
||||
|
||||
|
||||
def percentile(values: list[float], p: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
sorted_values = sorted(values)
|
||||
k = (len(sorted_values) - 1) * p / 100.0
|
||||
f = int(k)
|
||||
c = min(f + 1, len(sorted_values) - 1)
|
||||
if f == c:
|
||||
return sorted_values[f]
|
||||
return sorted_values[f] * (c - k) + sorted_values[c] * (k - f)
|
||||
|
||||
|
||||
def parse_jsonl(jsonl_path: Path) -> tuple[list[dict], dict]:
|
||||
"""Parse sglang.bench_serving --output-file output.
|
||||
|
||||
Newer sglang writes one aggregate JSON object. Older versions write one
|
||||
JSON object per request. Returns (raw_requests, aggregates).
|
||||
"""
|
||||
text = jsonl_path.read_text(encoding="utf-8", errors="replace").strip()
|
||||
if not text:
|
||||
return [], {}
|
||||
|
||||
# Try single aggregate JSON object first.
|
||||
try:
|
||||
data = json.loads(text)
|
||||
if isinstance(data, dict) and "mean_e2e_latency_ms" in data:
|
||||
aggregates = {
|
||||
"success": data.get("total_output_tokens", 0) > 0 and 1 or 0,
|
||||
"failed": 0,
|
||||
"input_tokens": data.get("total_input_tokens", 0),
|
||||
"output_tokens": data.get("total_output_tokens", 0),
|
||||
"latencies": {
|
||||
"e2e_ms": {
|
||||
"mean": data.get("mean_e2e_latency_ms"),
|
||||
"p50": data.get("median_e2e_latency_ms"),
|
||||
"p90": data.get("p90_e2e_latency_ms"),
|
||||
"p95": None,
|
||||
"p99": data.get("p99_e2e_latency_ms"),
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": data.get("mean_ttft_ms"),
|
||||
"p50": data.get("median_ttft_ms"),
|
||||
"p90": None,
|
||||
"p95": None,
|
||||
"p99": data.get("p99_ttft_ms"),
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": data.get("mean_tpot_ms"),
|
||||
"p50": data.get("median_tpot_ms"),
|
||||
"p90": None,
|
||||
"p95": None,
|
||||
"p99": data.get("p99_tpot_ms"),
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": data.get("mean_itl_ms"),
|
||||
"p50": data.get("median_itl_ms"),
|
||||
"p90": None,
|
||||
"p95": data.get("p95_itl_ms"),
|
||||
"p99": data.get("p99_itl_ms"),
|
||||
},
|
||||
},
|
||||
}
|
||||
return [data], aggregates
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Fall back to JSONL per-request parsing.
|
||||
raw_requests = []
|
||||
ttfts = []
|
||||
tpots = []
|
||||
itls = []
|
||||
e2es = []
|
||||
input_tokens = []
|
||||
output_tokens = []
|
||||
success = 0
|
||||
failed = 0
|
||||
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
req = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
raw_requests.append(req)
|
||||
|
||||
ttft = req.get("ttft") or req.get("ttft_ms") or 0
|
||||
tpot = req.get("tpot") or req.get("tpot_ms") or 0
|
||||
itl = req.get("inter_token_latency") or req.get("itl") or req.get("itl_ms") or 0
|
||||
e2e = req.get("e2e_latency") or req.get("e2e") or req.get("e2e_ms") or 0
|
||||
in_tok = req.get("input_tokens") or req.get("prompt_tokens") or 0
|
||||
out_tok = req.get("output_tokens") or req.get("completion_tokens") or 0
|
||||
|
||||
if ttft:
|
||||
ttfts.append(float(ttft))
|
||||
if tpot:
|
||||
tpots.append(float(tpot))
|
||||
if itl:
|
||||
itls.append(float(itl))
|
||||
if e2e:
|
||||
e2es.append(float(e2e))
|
||||
if in_tok:
|
||||
input_tokens.append(int(in_tok))
|
||||
if out_tok:
|
||||
output_tokens.append(int(out_tok))
|
||||
|
||||
if req.get("success", True):
|
||||
success += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
def latency_bucket(values: list[float]) -> dict:
|
||||
if not values:
|
||||
return {"mean": None, "p50": None, "p90": None, "p95": None, "p99": None}
|
||||
return {
|
||||
"mean": round(sum(values) / len(values), 2),
|
||||
"p50": round(percentile(values, 50), 2),
|
||||
"p90": round(percentile(values, 90), 2),
|
||||
"p95": round(percentile(values, 95), 2),
|
||||
"p99": round(percentile(values, 99), 2),
|
||||
}
|
||||
|
||||
aggregates = {
|
||||
"success": success,
|
||||
"failed": failed,
|
||||
"input_tokens": sum(input_tokens),
|
||||
"output_tokens": sum(output_tokens),
|
||||
"latencies": {
|
||||
"e2e_ms": latency_bucket(e2es),
|
||||
"ttft_ms": latency_bucket(ttfts),
|
||||
"tpot_ms": latency_bucket(tpots),
|
||||
"itl_ms": latency_bucket(itls),
|
||||
},
|
||||
}
|
||||
|
||||
return raw_requests, aggregates
|
||||
|
||||
|
||||
def parse_scenario(result_root: Path, raw_file: Path) -> dict | None:
|
||||
"""Parse one raw output file into a scenario dict."""
|
||||
# Filename: {chip}_{engine}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl
|
||||
parts = raw_file.stem.split("_")
|
||||
if len(parts) < 6:
|
||||
return None
|
||||
try:
|
||||
concurrency = int(parts[-3])
|
||||
input_len = int(parts[-2])
|
||||
output_len = int(parts[-1])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
log_file = result_root / "logs" / f"sglang_c{concurrency}_i{input_len}_o{output_len}.log"
|
||||
summary = {}
|
||||
if log_file.exists():
|
||||
summary = parse_summary_log(log_file.read_text(encoding="utf-8", errors="replace"))
|
||||
|
||||
raw_requests, aggregates = parse_jsonl(raw_file)
|
||||
|
||||
# For newer sglang aggregate JSON, success/failed are not present in the
|
||||
# raw output file. Override them from the human-readable summary log when
|
||||
# it is available.
|
||||
if summary.get("successful_requests") is not None:
|
||||
aggregates["success"] = int(summary["successful_requests"])
|
||||
# The summary log only reports successes; assume failures are zero
|
||||
# unless the aggregate JSON already provided a non-zero failed count.
|
||||
if not aggregates.get("failed"):
|
||||
aggregates["failed"] = 0
|
||||
|
||||
scenario = {
|
||||
"name": f"c{concurrency}_i{input_len}_o{output_len}",
|
||||
"concurrency": concurrency,
|
||||
"input_len": input_len,
|
||||
"output_len": output_len,
|
||||
"success": aggregates["success"],
|
||||
"failed": aggregates["failed"],
|
||||
"duration_s": summary.get("benchmark_duration_s"),
|
||||
"request_throughput": summary.get("request_throughput"),
|
||||
"input_token_throughput": summary.get("input_token_throughput"),
|
||||
"output_token_throughput": summary.get("output_token_throughput"),
|
||||
"total_token_throughput": summary.get("total_token_throughput"),
|
||||
"accept_length": None,
|
||||
"latencies": aggregates["latencies"],
|
||||
"raw_requests": raw_requests[:100] if len(raw_requests) <= 100 else None,
|
||||
}
|
||||
|
||||
return scenario
|
||||
|
||||
|
||||
def main() -> None:
|
||||
result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results")
|
||||
raw_dir = result_root / "raw_outputs"
|
||||
json_path = result_root / "results.json"
|
||||
report_path = result_root / "report.md"
|
||||
|
||||
if not json_path.exists():
|
||||
raise SystemExit(f"metadata results.json not found: {json_path}")
|
||||
|
||||
with open(json_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
scenarios = []
|
||||
if raw_dir.exists():
|
||||
for raw_file in sorted(raw_dir.glob("*.jsonl")):
|
||||
scenario = parse_scenario(result_root, raw_file)
|
||||
if scenario:
|
||||
scenarios.append(scenario)
|
||||
|
||||
data["scenarios"] = scenarios
|
||||
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# Generate report.md
|
||||
with open(report_path, "w", encoding="utf-8") as f:
|
||||
meta = data["metadata"]
|
||||
f.write(f"# Benchmark Report: {meta['experiment']}\n\n")
|
||||
f.write("## Metadata\n\n")
|
||||
f.write(f"- **Run ID**: {meta['run_id']}\n")
|
||||
f.write(f"- **Timestamp**: {meta['timestamp']}\n")
|
||||
f.write(f"- **Chip/Accelerator**: {meta.get('accelerator', '')} / {meta.get('chip', '')}\n")
|
||||
f.write(f"- **Engine/Backend**: {meta.get('engine', '')} / {meta.get('backend', '')}\n")
|
||||
f.write(f"- **Hardware**: {meta.get('hardware', '')}\n")
|
||||
f.write(f"- **Model**: {meta['model']}\n")
|
||||
f.write(f"- **Git Commit**: {meta.get('git_commit', 'unknown')}\n")
|
||||
f.write("\n## Results\n\n")
|
||||
f.write(
|
||||
"| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | "
|
||||
"TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 |\n"
|
||||
)
|
||||
f.write(
|
||||
"|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n"
|
||||
)
|
||||
for s in scenarios:
|
||||
lat = s["latencies"]
|
||||
f.write(
|
||||
f"| {s['name']} "
|
||||
f"| {s['concurrency']} "
|
||||
f"| {s['input_len']}/{s['output_len']} "
|
||||
f"| {s['success']} "
|
||||
f"| {s['failed']} "
|
||||
f"| {s.get('request_throughput') or ''} "
|
||||
f"| {s.get('output_token_throughput') or ''} "
|
||||
f"| {lat['ttft_ms']['p50'] or ''} "
|
||||
f"| {lat['ttft_ms']['p99'] or ''} "
|
||||
f"| {lat['tpot_ms']['p50'] or ''} "
|
||||
f"| {lat['tpot_ms']['p99'] or ''} "
|
||||
f"| {lat['e2e_ms']['p99'] or ''} |\n"
|
||||
)
|
||||
|
||||
print(f"Updated {json_path}")
|
||||
print(f"Wrote {report_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,17 @@
|
||||
# Benchmark Report: qwen3_235b_p800_sglang_tp8
|
||||
|
||||
## Metadata
|
||||
|
||||
- **Run ID**: qwen3_235b_tp8_run1
|
||||
- **Timestamp**: 2026-07-21T06:03:12+00:00
|
||||
- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800
|
||||
- **Engine/Backend**: sglang-xpu / sglang
|
||||
- **Hardware**: 8x Kunlun P800 XPU
|
||||
- **Model**: /data1/models/Qwen3-235B-A22B
|
||||
- **Git Commit**: d60b5bc
|
||||
|
||||
## Results
|
||||
|
||||
| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| c16_i2048_o2048 | 16 | 2048/2048 | 160 | 0 | 0.21 | 424.98 | 3261.300305603072 | 6779.247232209891 | 36.0282028564145 | 38.118499132202196 | 81609.06106460141 |
|
||||
155
experiments/p800/qwen3_235b_p800_sglang_tp8/run_bench.sh
Executable file
155
experiments/p800/qwen3_235b_p800_sglang_tp8/run_bench.sh
Executable file
@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env bash
|
||||
# Orchestrator for the Qwen3-235B-A22B TP=8 benchmark on Kunlun P800.
|
||||
#
|
||||
# Flow: write metadata -> start server (start_server.sh) -> warmup+bench_serving
|
||||
# (isl=2048 osl=2048 c=16 n=160) -> parse results -> report.md.
|
||||
#
|
||||
# The bench client (sglang.bench_serving) runs INSIDE the container; results are
|
||||
# written to a /data1 path that is mounted same-path into the container, so the
|
||||
# host-side parse_results.py can read them directly.
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||
# Source config.env BEFORE platform.sh so our CONTAINER_NAME default wins
|
||||
# over the platform's "sglang-dsv4-flash" default (avoids name collisions
|
||||
# with the dsv4 experiments, which would docker rm -f this container).
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/config.env"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||
|
||||
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
|
||||
RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}"
|
||||
RAW_DIR="${RESULT_ROOT}/raw_outputs"
|
||||
LOG_DIR="${RESULT_ROOT}/logs"
|
||||
mkdir -p "$RAW_DIR" "$LOG_DIR"
|
||||
|
||||
ensure_result_root "$RESULT_ROOT"
|
||||
log_init "${LOG_DIR}/orchestrator.log"
|
||||
|
||||
log "experiment=${EXPERIMENT_NAME}"
|
||||
log "run_id=${RUN_ID}"
|
||||
log "result_root=${RESULT_ROOT}"
|
||||
log "platform=${PLATFORM} chip=${CHIP} hardware=${HARDWARE}"
|
||||
log "model=${MODEL_PATH} tp=${TP} backend=${BACKEND}"
|
||||
log "scenarios=${SCENARIOS[*]}"
|
||||
|
||||
# --- metadata --------------------------------------------------------------
|
||||
METADATA_JSON="${RESULT_ROOT}/results.json"
|
||||
write_metadata_json \
|
||||
"$METADATA_JSON" \
|
||||
"$EXPERIMENT_NAME" \
|
||||
"$RUN_ID" \
|
||||
"$MODEL_PATH" \
|
||||
"$BACKEND" \
|
||||
"$ENGINE" \
|
||||
"$HARDWARE" \
|
||||
"$ACCELERATOR" \
|
||||
"$CHIP" \
|
||||
"experiments/${EXPERIMENT_NAME}/run_bench.sh" \
|
||||
"" \
|
||||
"Qwen3-235B-A22B SGLang TP=8 benchmark on Kunlun P800 (isl=osl=2048, c=16)"
|
||||
|
||||
"${PYTHON:-python3}" - "$METADATA_JSON" "$TP" "$CONTEXT_LENGTH" "$PORT" "$CONTAINER_NAME" "$DOCKER_IMAGE" "$MEM_FRACTION_STATIC" "$DTYPE" "$ATTENTION_BACKEND" <<'PY'
|
||||
import json, sys
|
||||
p, tp, ctx, port, cont, img, memfrac, dtype, attn = sys.argv[1:10]
|
||||
with open(p) as f:
|
||||
d = json.load(f)
|
||||
d["config"] = {
|
||||
"tp": int(tp),
|
||||
"context_length": int(ctx),
|
||||
"port": int(port),
|
||||
"container": cont,
|
||||
"docker_image": img,
|
||||
"attention_backend": attn,
|
||||
"dtype": dtype,
|
||||
"mem_fraction_static": float(memfrac),
|
||||
"max_running_requests": 16,
|
||||
"page_size": 64,
|
||||
"benchmark": {"input_len": 2048, "output_len": 2048, "concurrency": 16, "num_prompts": 160},
|
||||
}
|
||||
with open(p, "w") as f:
|
||||
json.dump(d, f, indent=2, ensure_ascii=False)
|
||||
PY
|
||||
|
||||
# --- start server (unless caller manages it themselves) --------------------
|
||||
if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then
|
||||
log "SKIP_MANAGE_SERVER set; assuming server already running on port ${PORT}"
|
||||
if ! health_check 127.0.0.1 "$PORT" 30; then
|
||||
log "ERROR: no healthy server at port ${PORT}"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
log "starting server via start_server.sh"
|
||||
bash "${SCRIPT_DIR}/start_server.sh" >> "${LOG_DIR}/start_server.outer.log" 2>&1
|
||||
fi
|
||||
|
||||
on_exit() {
|
||||
local code=$?
|
||||
if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then
|
||||
log "stopping sglang inside ${CONTAINER_NAME} (container kept alive)"
|
||||
docker exec "$CONTAINER_NAME" bash -lc 'pkill -9 -f sglang.launch_server 2>/dev/null || true' || true
|
||||
fi
|
||||
log "orchestrator exiting with code=${code}"
|
||||
exit "$code"
|
||||
}
|
||||
trap on_exit EXIT
|
||||
|
||||
# --- run scenarios ---------------------------------------------------------
|
||||
log "===== BENCHMARK START ====="
|
||||
|
||||
for scenario in "${SCENARIOS[@]}"; do
|
||||
scenario="${scenario//[()]/}"
|
||||
read -r concurrency input_len output_len num_prompts <<< "$scenario"
|
||||
|
||||
today="$(date '+%m%d')"
|
||||
# Filename convention expected by parse_results.py:
|
||||
# {chip}_{engine}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl
|
||||
output_file="${RAW_DIR}/${CHIP}_${ENGINE}_${today}_${concurrency}_${input_len}_${output_len}.jsonl"
|
||||
# parse_results.py reads the human-readable summary from this exact path.
|
||||
summary_log="${LOG_DIR}/sglang_c${concurrency}_i${input_len}_o${output_len}.log"
|
||||
|
||||
log "running scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}"
|
||||
log "output_file=${output_file}"
|
||||
|
||||
docker exec "$CONTAINER_NAME" bash -lc "
|
||||
source /root/miniconda/etc/profile.d/conda.sh
|
||||
conda activate python310_torch25_cuda
|
||||
python -m sglang.bench_serving \
|
||||
--backend ${BACKEND} \
|
||||
--host 127.0.0.1 \
|
||||
--port ${PORT} \
|
||||
--model ${MODEL_PATH} \
|
||||
--dataset-name random \
|
||||
--dataset-path ${DATASET_PATH} \
|
||||
--random-input-len ${input_len} \
|
||||
--random-output-len ${output_len} \
|
||||
--random-range-ratio ${RANDOM_RANGE_RATIO} \
|
||||
--num-prompts ${num_prompts} \
|
||||
--warmup-requests ${WARMUP} \
|
||||
--max-concurrency ${concurrency} \
|
||||
--request-rate ${REQUEST_RATE} \
|
||||
--output-file ${output_file} \
|
||||
--output-details \
|
||||
--seed ${SEED}
|
||||
" > "$summary_log" 2>&1 || {
|
||||
log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${summary_log}"
|
||||
tail -60 "$summary_log" 2>/dev/null || true
|
||||
continue
|
||||
}
|
||||
|
||||
log "finished scenario: output=${output_file}"
|
||||
log "----- summary -----"
|
||||
tail -40 "$summary_log" | sed 's/^/ /'
|
||||
done
|
||||
|
||||
log "===== BENCHMARK DONE ====="
|
||||
log "parsing results..."
|
||||
"${PYTHON:-python3}" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" || log "WARNING: parser failed"
|
||||
|
||||
log "all results saved to ${RESULT_ROOT}"
|
||||
log "report: ${RESULT_ROOT}/report.md"
|
||||
151
experiments/p800/qwen3_235b_p800_sglang_tp8/start_server.sh
Executable file
151
experiments/p800/qwen3_235b_p800_sglang_tp8/start_server.sh
Executable file
@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env bash
|
||||
# Start the SGLang server for Qwen3-235B-A22B on Kunlun P800 (TP=8, 8x XPU).
|
||||
#
|
||||
# Adapted from the proven qwen3-8b TP=1 launch on this image. Differences:
|
||||
# - new container exposing all 8 /dev/xpu* devices (the 8b container only had xpu0)
|
||||
# - --tp-size 8, XPU_VISIBLE_DEVICES=0..7
|
||||
# - context-length 8192 (8b used 4096)
|
||||
#
|
||||
# All XPU-specific env vars (XSGL_*, XMLIR_*) are taken verbatim from the 8b run.
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||
# Source config.env BEFORE platform.sh so our CONTAINER_NAME default wins
|
||||
# over the platform's "sglang-dsv4-flash" default (avoids name collisions
|
||||
# with the dsv4 experiments, which would docker rm -f this container).
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/config.env"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||
|
||||
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
|
||||
RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}"
|
||||
LOG_DIR="${RESULT_ROOT}/logs"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
log_init "${LOG_DIR}/start_server.log"
|
||||
log "starting server for ${EXPERIMENT_NAME}"
|
||||
log "model: ${MODEL_PATH}"
|
||||
log "tp: ${TP} xpu: ${XPU_VISIBLE_DEVICES}"
|
||||
log "port: ${PORT}"
|
||||
log "container: ${CONTAINER_NAME}"
|
||||
log "image: ${DOCKER_IMAGE}"
|
||||
|
||||
SERVER_LOG="${LOG_DIR}/server.log"
|
||||
|
||||
# --- 1. Ensure the container is running with all 8 XPU devices -------------
|
||||
ensure_container() {
|
||||
if docker inspect "$CONTAINER_NAME" >/dev/null 2>&1; then
|
||||
log "container ${CONTAINER_NAME} exists; ensuring it is running"
|
||||
docker start "$CONTAINER_NAME" >/dev/null 2>&1 || true
|
||||
else
|
||||
log "creating container ${CONTAINER_NAME} from ${DOCKER_IMAGE}"
|
||||
local device_args=""
|
||||
local i
|
||||
for i in 0 1 2 3 4 5 6 7; do
|
||||
device_args="${device_args} --device /dev/xpu${i}:/dev/xpu${i}"
|
||||
done
|
||||
device_args="${device_args} --device /dev/xpuctrl:/dev/xpuctrl"
|
||||
|
||||
docker run -d \
|
||||
--name "$CONTAINER_NAME" \
|
||||
--privileged \
|
||||
--network host \
|
||||
--ipc host \
|
||||
--security-opt label=disable \
|
||||
--shm-size 1g \
|
||||
${device_args} \
|
||||
-v /data1:/data1 \
|
||||
-e XPU_VISIBLE_DEVICES="${XPU_VISIBLE_DEVICES}" \
|
||||
-e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
|
||||
-e TZ=Asia/Shanghai \
|
||||
--entrypoint /bin/bash \
|
||||
"$DOCKER_IMAGE" \
|
||||
-lc "source /etc/profile >/dev/null 2>&1 || true; exec sleep infinity"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- 2. Launch sglang.launch_server inside the container -------------------
|
||||
launch_sglang() {
|
||||
log "launching sglang launch_server (TP=${TP}) inside ${CONTAINER_NAME}"
|
||||
# Kill any prior sglang process inside the container.
|
||||
docker exec "$CONTAINER_NAME" bash -lc 'pkill -9 -f sglang.launch_server 2>/dev/null || true' || true
|
||||
|
||||
docker exec -d "$CONTAINER_NAME" bash -lc "
|
||||
source /root/miniconda/etc/profile.d/conda.sh
|
||||
conda activate python310_torch25_cuda
|
||||
export XPU_VISIBLE_DEVICES=${XPU_VISIBLE_DEVICES}
|
||||
export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}
|
||||
export XSGL_INTERTYPE_BFP16=1
|
||||
export XSGL_FAST_SWIGLU=1
|
||||
export XMLIR_FORCE_USE_XPU_GRAPH=1
|
||||
export XSGL_FUSE_SPLIT_NORM_ROPE_NEOX=1
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3000000
|
||||
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=600
|
||||
exec python -m sglang.launch_server \
|
||||
--model-path ${MODEL_PATH} \
|
||||
--host 0.0.0.0 \
|
||||
--port ${PORT} \
|
||||
--tp-size ${TP} \
|
||||
--context-length ${CONTEXT_LENGTH} \
|
||||
--max-running-requests ${MAX_RUNNING_REQUESTS} \
|
||||
--attention-backend ${ATTENTION_BACKEND} \
|
||||
--trust-remote-code \
|
||||
--disable-radix-cache \
|
||||
--disable-custom-all-reduce \
|
||||
--chunked-prefill-size -1 \
|
||||
--page-size ${PAGE_SIZE} \
|
||||
--mem-fraction-static ${MEM_FRACTION_STATIC} \
|
||||
--max-prefill-tokens ${MAX_PREFILL_TOKENS} \
|
||||
--dtype ${DTYPE} \
|
||||
--cuda-graph-max-bs ${CUDA_GRAPH_MAX_BS} \
|
||||
--cuda-graph-bs ${CUDA_GRAPH_BS} \
|
||||
--watchdog-timeout 3000000 \
|
||||
> ${SERVER_LOG} 2>&1
|
||||
"
|
||||
}
|
||||
|
||||
ensure_container
|
||||
|
||||
# --- 1b. Patch qwen3_moe.py bug in the image -------------------------------
|
||||
# The Kunlun sglang image's qwen3_moe.py:1829 assigns to the read-only
|
||||
# @property `routed_experts_weights_of_layer` instead of the private
|
||||
# `_routed_experts_weights_of_layer`, raising
|
||||
# AttributeError: can't set attribute 'routed_experts_weights_of_layer'
|
||||
# for any qwen3 MoE model (e.g. Qwen3-235B-A22B). Qwen3-8B is dense, so the
|
||||
# proven 8b run never hit this path. Fix: assign to the private attr.
|
||||
# Idempotent - safe to run on every start.
|
||||
patch_qwen3_moe() {
|
||||
log "applying qwen3_moe.py patch in ${CONTAINER_NAME}"
|
||||
docker exec "$CONTAINER_NAME" bash -lc '
|
||||
F=/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/sglang/srt/models/qwen3_moe.py
|
||||
if grep -q "self\.routed_experts_weights_of_layer = LazyValue(" "$F"; then
|
||||
cp "$F" "$F.bak.qwen3moe" 2>/dev/null || true
|
||||
sed -i "s/self\.routed_experts_weights_of_layer = LazyValue(/self._routed_experts_weights_of_layer = LazyValue(/" "$F"
|
||||
echo "PATCHED qwen3_moe.py (routed_experts_weights_of_layer -> _routed_experts_weights_of_layer)"
|
||||
else
|
||||
echo "qwen3_moe.py already patched"
|
||||
fi
|
||||
'
|
||||
}
|
||||
patch_qwen3_moe
|
||||
|
||||
launch_sglang
|
||||
|
||||
# 235B weights (~470GB) + cuda-graph capture for 16 batch sizes take a while.
|
||||
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-1200}"
|
||||
log "waiting for health on http://127.0.0.1:${PORT}/health (timeout=${HEALTH_TIMEOUT}s)"
|
||||
if health_check 127.0.0.1 "$PORT" "$HEALTH_TIMEOUT"; then
|
||||
log "sglang server is READY at http://127.0.0.1:${PORT}"
|
||||
log "server log: ${SERVER_LOG} (or: docker logs -f ${CONTAINER_NAME})"
|
||||
exit 0
|
||||
else
|
||||
log "ERROR: server not healthy after ${HEALTH_TIMEOUT}s"
|
||||
log "----- last 200 lines of server log -----"
|
||||
tail -200 "$SERVER_LOG" 2>/dev/null || docker logs --tail 200 "$CONTAINER_NAME" 2>&1 | tail -200
|
||||
exit 1
|
||||
fi
|
||||
Loading…
x
Reference in New Issue
Block a user