diff --git a/experiments/p800/qwen3_235b_p800_sglang_tp8/README.md b/experiments/p800/qwen3_235b_p800_sglang_tp8/README.md new file mode 100644 index 0000000..c35adb3 --- /dev/null +++ b/experiments/p800/qwen3_235b_p800_sglang_tp8/README.md @@ -0,0 +1,63 @@ +# Qwen3-235B-A22B · Kunlun P800 · SGLang TP=8 + +在 p800.2（8× 昆仑芯 P800 XPU，每卡 96GB）上用 SGLang 以 **TP=8** 部署 +`Qwen3-235B-A22B`，并用 `sglang.bench_serving` 跑 **输入 2048 / 输出 2048 / 并发 16** +的吞吐基准，输出吞吐、TTFT、TPOT、ITL、E2E 等指标。 + +## 关键设计 + +本实验借鉴仓库里已验证可用的 **qwen3-8b TP=1** 启动命令（同一昆仑 P800 sglang 镜像）， +仅做以下适配： + +| 项 | qwen3-8b (TP=1) | 本实验 (TP=8) | +|---|---|---| +| 容器 XPU | 仅 `/dev/xpu0` | `/dev/xpu0..7`（新容器） | +| `--tp-size` | 1 | 8 | +| `XPU_VISIBLE_DEVICES` | 0 | 0,1,2,3,4,5,6,7 | +| `--model-path` | Qwen3-8B | Qwen3-235B-A22B | +| `--context-length` | 4096 | 8192（8b 用 4096 在 2k+2k 下截断了 2 条输出） | +| 其余启动参数 | — | 完全一致 | + +不变的 Qwen3 专用配置（与 DSV4 实验不同）： +- `--attention-backend kunlun`（非 DSV4 的 `nsa`） +- XPU 环境变量：`XSGL_INTERTYPE_BFP16=1`、`XSGL_FAST_SWIGLU=1`、 + `XMLIR_FORCE_USE_XPU_GRAPH=1`、`XSGL_FUSE_SPLIT_NORM_ROPE_NEOX=1` +- `--dtype float16`、`--disable-radix-cache`、`--chunked-prefill-size -1`、 + `--page-size 64`、`--mem-fraction-static 0.9`、`--cuda-graph-bs 1..16` +- 无 DSV4 专用变量（`IS_DSV4`/`SGLANG_DSV4_MODE` 等）、无 reasoning/tool-call parser + +显存预算：235B FP16 权重 ~470GB，TP=8 每卡 ~59GB；每卡 96GB，0.9 静态占比留 +~27GB/卡 给 KV+激活+cuda graph，2k+2k 上下文 KV 极小，充裕。 + +> 注意：框架 `scripts/common/server_docker.sh` 里的 `docker_server_start` 是 +> DeepSeek-V4 硬编码的（NSA attention、`IS_DSV4=1`、deepseekv4 parser）， +> **不能**用于 Qwen3，因此本实验自带独立的 `start_server.sh`。 + +## 文件 + +- `config.env` — 实验参数唯一来源 +- `start_server.sh` — 起 8-XPU 容器 + 容器内启动 sglang + 健康检查 +- `run_bench.sh` — 编排入口（写元数据 → 起服务 → warmup+bench → 解析） +- `parse_results.py` — 解析 `sglang.bench_serving` 输出（复用自 dsv4_p800_sglang） +- `results//` — 自动生成：`raw_outputs/*.jsonl`、`logs/`、`results.json`、`report.md` + +## 用法 + +```bash +cd /data1/yy/sskj/experiments/p800/qwen3_235b_p800_sglang_tp8 + +# 一键跑（起服务 + 基准 + 解析） +bash run_bench.sh + +# 或分步：先起服务（便于排查），再复用服务跑基准 +bash start_server.sh +SKIP_MANAGE_SERVER=1 bash run_bench.sh +``` + +服务启动加载 235B 权重（~470GB）+ 捕获 16 个 batch size 的 cuda graph 较慢， +健康检查超时设为 1200s。结果在 `results//report.md`。 + +## 输出指标（report.md） + +每个 scenario 给出：Success/Failed、Req/s、Input/Output/Total tok/s、 +TTFT（mean/p50/p90/p99）、TPOT（mean/p50/p90/p99）、E2E（p99）等。 diff --git a/experiments/p800/qwen3_235b_p800_sglang_tp8/config.env b/experiments/p800/qwen3_235b_p800_sglang_tp8/config.env new file mode 100644 index 0000000..72ad149 --- /dev/null +++ b/experiments/p800/qwen3_235b_p800_sglang_tp8/config.env @@ -0,0 +1,51 @@ +# Experiment: Qwen3-235B-A22B on Kunlun P800 (8x XPU), SGLang, TP=8. +# Benchmark: input 2048 / output 2048, concurrency 16, num_prompts 160. +# +# Adapted from the proven qwen3-8b TP=1 launch on this same image +# (attention-backend kunlun, XSGL_* XPU env vars, --dtype float16). +# Only differences vs the 8b run: --tp-size 8, 8 XPU devices, model path, +# and context-length 8192 (8b used 4096 which truncated 2 outputs at 2k+2k). + +EXPERIMENT="${EXPERIMENT:-qwen3_235b_p800_sglang_tp8}" +MODEL_NAME="${MODEL_NAME:-Qwen3-235B-A22B}" +MODEL_PATH="${MODEL_PATH:-/data1/models/Qwen3-235B-A22B}" +SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-/data1/models/Qwen3-235B-A22B}" +BACKEND="${BACKEND:-sglang}" +ENGINE="${ENGINE:-sglang-xpu}" +DATASET="${DATASET:-random}" +DATASET_PATH="${DATASET_PATH:-/data1/yy/sharegpt_data/ShareGPT_V3_unfiltered_cleaned_split.json}" + +# Serving port + container. +# 30000 is taken by the lingering qwen3_8b_bench_tp1 container (docker-proxy), +# so default to 30010. +PORT="${PORT:-30010}" +CONTAINER_NAME="${CONTAINER_NAME:-qwen3_235b_bench_tp8}" + +# Docker image: Kunlun P800 SGLang image (same one the qwen3-8b run used). +DOCKER_IMAGE="${DOCKER_IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202}" + +# Hardware: 8x Kunlun P800 XPU, TP=8. +TP="${TP:-8}" +XPU_VISIBLE_DEVICES="${XPU_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" + +# SGLang launch parameters (mirrors the working qwen3-8b config, TP bumped to 8). +CONTEXT_LENGTH="${CONTEXT_LENGTH:-8192}" +MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-16}" +ATTENTION_BACKEND="${ATTENTION_BACKEND:-kunlun}" +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.9}" +DTYPE="${DTYPE:-float16}" +PAGE_SIZE="${PAGE_SIZE:-64}" +MAX_PREFILL_TOKENS="${MAX_PREFILL_TOKENS:-32768}" +CUDA_GRAPH_MAX_BS="${CUDA_GRAPH_MAX_BS:-16}" +CUDA_GRAPH_BS="${CUDA_GRAPH_BS:-1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}" + +# Bench scenario: "concurrency input_len output_len num_prompts". +SCENARIOS=( + "16 2048 2048 160" +) + +WARMUP="${WARMUP:-16}" +REQUEST_RATE="${REQUEST_RATE:-10000}" +SEED="${SEED:-1}" +RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1}" diff --git a/experiments/p800/qwen3_235b_p800_sglang_tp8/parse_results.py b/experiments/p800/qwen3_235b_p800_sglang_tp8/parse_results.py new file mode 100755 index 0000000..341f69e --- /dev/null +++ b/experiments/p800/qwen3_235b_p800_sglang_tp8/parse_results.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""Parse P800 SGLang benchmark outputs and produce results.json + report.md. + +Usage: + python3 parse_results.py +""" + +import json +import os +import re +import sys +from pathlib import Path +from collections import OrderedDict + +METRIC_PATTERNS = OrderedDict( + [ + ("successful_requests", [r"Successful requests:\s+(\d+)"]), + ("benchmark_duration_s", [r"Benchmark duration \(s\):\s+([\d.]+)"]), + ("request_throughput", [r"Request throughput \(req/s\):\s+([\d.]+)"]), + ("input_token_throughput", [r"Input token throughput \(tok/s\):\s+([\d.]+)"]), + ("output_token_throughput", [r"Output token throughput \(tok/s\):\s+([\d.]+)"]), + ("total_token_throughput", [r"Total token throughput \(tok/s\):\s+([\d.]+)"]), + ("ttft_mean", [r"Mean TTFT \(ms\):\s+([\d.]+)"]), + ("ttft_p50", [r"Median TTFT \(ms\):\s+([\d.]+)", r"P50 TTFT \(ms\):\s+([\d.]+)"]), + ("ttft_p90", [r"P90 TTFT \(ms\):\s+([\d.]+)"]), + ("ttft_p99", [r"P99 TTFT \(ms\):\s+([\d.]+)"]), + ("tpot_mean", [r"Mean TPOT \(ms\):\s+([\d.]+)"]), + ("tpot_p50", [r"Median TPOT \(ms\):\s+([\d.]+)", r"P50 TPOT \(ms\):\s+([\d.]+)"]), + ("tpot_p90", [r"P90 TPOT \(ms\):\s+([\d.]+)"]), + ("tpot_p99", [r"P99 TPOT \(ms\):\s+([\d.]+)"]), + ("e2e_mean", [r"Mean E2E \(ms\):\s+([\d.]+)"]), + ("e2e_p50", [r"Median E2E \(ms\):\s+([\d.]+)", r"P50 E2E \(ms\):\s+([\d.]+)"]), + ("e2e_p90", [r"P90 E2E \(ms\):\s+([\d.]+)"]), + ("e2e_p99", [r"P99 E2E \(ms\):\s+([\d.]+)"]), + ] +) + + +def find_float(text: str, patterns: list[str]) -> float | None: + for pat in patterns: + m = re.search(pat, text) + if m: + try: + return float(m.group(1)) + except ValueError: + return None + return None + + +def parse_summary_log(log_text: str) -> dict: + result = {} + for key, patterns in METRIC_PATTERNS.items(): + result[key] = find_float(log_text, patterns) + return result + + +def percentile(values: list[float], p: float) -> float: + if not values: + return 0.0 + sorted_values = sorted(values) + k = (len(sorted_values) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(sorted_values) - 1) + if f == c: + return sorted_values[f] + return sorted_values[f] * (c - k) + sorted_values[c] * (k - f) + + +def parse_jsonl(jsonl_path: Path) -> tuple[list[dict], dict]: + """Parse sglang.bench_serving --output-file output. + + Newer sglang writes one aggregate JSON object. Older versions write one + JSON object per request. Returns (raw_requests, aggregates). + """ + text = jsonl_path.read_text(encoding="utf-8", errors="replace").strip() + if not text: + return [], {} + + # Try single aggregate JSON object first. + try: + data = json.loads(text) + if isinstance(data, dict) and "mean_e2e_latency_ms" in data: + aggregates = { + "success": data.get("total_output_tokens", 0) > 0 and 1 or 0, + "failed": 0, + "input_tokens": data.get("total_input_tokens", 0), + "output_tokens": data.get("total_output_tokens", 0), + "latencies": { + "e2e_ms": { + "mean": data.get("mean_e2e_latency_ms"), + "p50": data.get("median_e2e_latency_ms"), + "p90": data.get("p90_e2e_latency_ms"), + "p95": None, + "p99": data.get("p99_e2e_latency_ms"), + }, + "ttft_ms": { + "mean": data.get("mean_ttft_ms"), + "p50": data.get("median_ttft_ms"), + "p90": None, + "p95": None, + "p99": data.get("p99_ttft_ms"), + }, + "tpot_ms": { + "mean": data.get("mean_tpot_ms"), + "p50": data.get("median_tpot_ms"), + "p90": None, + "p95": None, + "p99": data.get("p99_tpot_ms"), + }, + "itl_ms": { + "mean": data.get("mean_itl_ms"), + "p50": data.get("median_itl_ms"), + "p90": None, + "p95": data.get("p95_itl_ms"), + "p99": data.get("p99_itl_ms"), + }, + }, + } + return [data], aggregates + except json.JSONDecodeError: + pass + + # Fall back to JSONL per-request parsing. + raw_requests = [] + ttfts = [] + tpots = [] + itls = [] + e2es = [] + input_tokens = [] + output_tokens = [] + success = 0 + failed = 0 + + for line in text.splitlines(): + line = line.strip() + if not line: + continue + try: + req = json.loads(line) + except json.JSONDecodeError: + continue + + raw_requests.append(req) + + ttft = req.get("ttft") or req.get("ttft_ms") or 0 + tpot = req.get("tpot") or req.get("tpot_ms") or 0 + itl = req.get("inter_token_latency") or req.get("itl") or req.get("itl_ms") or 0 + e2e = req.get("e2e_latency") or req.get("e2e") or req.get("e2e_ms") or 0 + in_tok = req.get("input_tokens") or req.get("prompt_tokens") or 0 + out_tok = req.get("output_tokens") or req.get("completion_tokens") or 0 + + if ttft: + ttfts.append(float(ttft)) + if tpot: + tpots.append(float(tpot)) + if itl: + itls.append(float(itl)) + if e2e: + e2es.append(float(e2e)) + if in_tok: + input_tokens.append(int(in_tok)) + if out_tok: + output_tokens.append(int(out_tok)) + + if req.get("success", True): + success += 1 + else: + failed += 1 + + def latency_bucket(values: list[float]) -> dict: + if not values: + return {"mean": None, "p50": None, "p90": None, "p95": None, "p99": None} + return { + "mean": round(sum(values) / len(values), 2), + "p50": round(percentile(values, 50), 2), + "p90": round(percentile(values, 90), 2), + "p95": round(percentile(values, 95), 2), + "p99": round(percentile(values, 99), 2), + } + + aggregates = { + "success": success, + "failed": failed, + "input_tokens": sum(input_tokens), + "output_tokens": sum(output_tokens), + "latencies": { + "e2e_ms": latency_bucket(e2es), + "ttft_ms": latency_bucket(ttfts), + "tpot_ms": latency_bucket(tpots), + "itl_ms": latency_bucket(itls), + }, + } + + return raw_requests, aggregates + + +def parse_scenario(result_root: Path, raw_file: Path) -> dict | None: + """Parse one raw output file into a scenario dict.""" + # Filename: {chip}_{engine}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl + parts = raw_file.stem.split("_") + if len(parts) < 6: + return None + try: + concurrency = int(parts[-3]) + input_len = int(parts[-2]) + output_len = int(parts[-1]) + except ValueError: + return None + + log_file = result_root / "logs" / f"sglang_c{concurrency}_i{input_len}_o{output_len}.log" + summary = {} + if log_file.exists(): + summary = parse_summary_log(log_file.read_text(encoding="utf-8", errors="replace")) + + raw_requests, aggregates = parse_jsonl(raw_file) + + # For newer sglang aggregate JSON, success/failed are not present in the + # raw output file. Override them from the human-readable summary log when + # it is available. + if summary.get("successful_requests") is not None: + aggregates["success"] = int(summary["successful_requests"]) + # The summary log only reports successes; assume failures are zero + # unless the aggregate JSON already provided a non-zero failed count. + if not aggregates.get("failed"): + aggregates["failed"] = 0 + + scenario = { + "name": f"c{concurrency}_i{input_len}_o{output_len}", + "concurrency": concurrency, + "input_len": input_len, + "output_len": output_len, + "success": aggregates["success"], + "failed": aggregates["failed"], + "duration_s": summary.get("benchmark_duration_s"), + "request_throughput": summary.get("request_throughput"), + "input_token_throughput": summary.get("input_token_throughput"), + "output_token_throughput": summary.get("output_token_throughput"), + "total_token_throughput": summary.get("total_token_throughput"), + "accept_length": None, + "latencies": aggregates["latencies"], + "raw_requests": raw_requests[:100] if len(raw_requests) <= 100 else None, + } + + return scenario + + +def main() -> None: + result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") + raw_dir = result_root / "raw_outputs" + json_path = result_root / "results.json" + report_path = result_root / "report.md" + + if not json_path.exists(): + raise SystemExit(f"metadata results.json not found: {json_path}") + + with open(json_path, "r", encoding="utf-8") as f: + data = json.load(f) + + scenarios = [] + if raw_dir.exists(): + for raw_file in sorted(raw_dir.glob("*.jsonl")): + scenario = parse_scenario(result_root, raw_file) + if scenario: + scenarios.append(scenario) + + data["scenarios"] = scenarios + + with open(json_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + # Generate report.md + with open(report_path, "w", encoding="utf-8") as f: + meta = data["metadata"] + f.write(f"# Benchmark Report: {meta['experiment']}\n\n") + f.write("## Metadata\n\n") + f.write(f"- **Run ID**: {meta['run_id']}\n") + f.write(f"- **Timestamp**: {meta['timestamp']}\n") + f.write(f"- **Chip/Accelerator**: {meta.get('accelerator', '')} / {meta.get('chip', '')}\n") + f.write(f"- **Engine/Backend**: {meta.get('engine', '')} / {meta.get('backend', '')}\n") + f.write(f"- **Hardware**: {meta.get('hardware', '')}\n") + f.write(f"- **Model**: {meta['model']}\n") + f.write(f"- **Git Commit**: {meta.get('git_commit', 'unknown')}\n") + f.write("\n## Results\n\n") + f.write( + "| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | " + "TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 |\n" + ) + f.write( + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n" + ) + for s in scenarios: + lat = s["latencies"] + f.write( + f"| {s['name']} " + f"| {s['concurrency']} " + f"| {s['input_len']}/{s['output_len']} " + f"| {s['success']} " + f"| {s['failed']} " + f"| {s.get('request_throughput') or ''} " + f"| {s.get('output_token_throughput') or ''} " + f"| {lat['ttft_ms']['p50'] or ''} " + f"| {lat['ttft_ms']['p99'] or ''} " + f"| {lat['tpot_ms']['p50'] or ''} " + f"| {lat['tpot_ms']['p99'] or ''} " + f"| {lat['e2e_ms']['p99'] or ''} |\n" + ) + + print(f"Updated {json_path}") + print(f"Wrote {report_path}") + + +if __name__ == "__main__": + main() diff --git a/experiments/p800/qwen3_235b_p800_sglang_tp8/results/qwen3_235b_tp8_run1/report.md b/experiments/p800/qwen3_235b_p800_sglang_tp8/results/qwen3_235b_tp8_run1/report.md new file mode 100644 index 0000000..56b9d1b --- /dev/null +++ b/experiments/p800/qwen3_235b_p800_sglang_tp8/results/qwen3_235b_tp8_run1/report.md @@ -0,0 +1,17 @@ +# Benchmark Report: qwen3_235b_p800_sglang_tp8 + +## Metadata + +- **Run ID**: qwen3_235b_tp8_run1 +- **Timestamp**: 2026-07-21T06:03:12+00:00 +- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 +- **Engine/Backend**: sglang-xpu / sglang +- **Hardware**: 8x Kunlun P800 XPU +- **Model**: /data1/models/Qwen3-235B-A22B +- **Git Commit**: d60b5bc + +## Results + +| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c16_i2048_o2048 | 16 | 2048/2048 | 160 | 0 | 0.21 | 424.98 | 3261.300305603072 | 6779.247232209891 | 36.0282028564145 | 38.118499132202196 | 81609.06106460141 | diff --git a/experiments/p800/qwen3_235b_p800_sglang_tp8/run_bench.sh b/experiments/p800/qwen3_235b_p800_sglang_tp8/run_bench.sh new file mode 100755 index 0000000..6c63f76 --- /dev/null +++ b/experiments/p800/qwen3_235b_p800_sglang_tp8/run_bench.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# Orchestrator for the Qwen3-235B-A22B TP=8 benchmark on Kunlun P800. +# +# Flow: write metadata -> start server (start_server.sh) -> warmup+bench_serving +# (isl=2048 osl=2048 c=16 n=160) -> parse results -> report.md. +# +# The bench client (sglang.bench_serving) runs INSIDE the container; results are +# written to a /data1 path that is mounted same-path into the container, so the +# host-side parse_results.py can read them directly. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# Source config.env BEFORE platform.sh so our CONTAINER_NAME default wins +# over the platform's "sglang-dsv4-flash" default (avoids name collisions +# with the dsv4 experiments, which would docker rm -f this container). +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" +RAW_DIR="${RESULT_ROOT}/raw_outputs" +LOG_DIR="${RESULT_ROOT}/logs" +mkdir -p "$RAW_DIR" "$LOG_DIR" + +ensure_result_root "$RESULT_ROOT" +log_init "${LOG_DIR}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "result_root=${RESULT_ROOT}" +log "platform=${PLATFORM} chip=${CHIP} hardware=${HARDWARE}" +log "model=${MODEL_PATH} tp=${TP} backend=${BACKEND}" +log "scenarios=${SCENARIOS[*]}" + +# --- metadata -------------------------------------------------------------- +METADATA_JSON="${RESULT_ROOT}/results.json" +write_metadata_json \ + "$METADATA_JSON" \ + "$EXPERIMENT_NAME" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "$BACKEND" \ + "$ENGINE" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "" \ + "Qwen3-235B-A22B SGLang TP=8 benchmark on Kunlun P800 (isl=osl=2048, c=16)" + +"${PYTHON:-python3}" - "$METADATA_JSON" "$TP" "$CONTEXT_LENGTH" "$PORT" "$CONTAINER_NAME" "$DOCKER_IMAGE" "$MEM_FRACTION_STATIC" "$DTYPE" "$ATTENTION_BACKEND" <<'PY' +import json, sys +p, tp, ctx, port, cont, img, memfrac, dtype, attn = sys.argv[1:10] +with open(p) as f: + d = json.load(f) +d["config"] = { + "tp": int(tp), + "context_length": int(ctx), + "port": int(port), + "container": cont, + "docker_image": img, + "attention_backend": attn, + "dtype": dtype, + "mem_fraction_static": float(memfrac), + "max_running_requests": 16, + "page_size": 64, + "benchmark": {"input_len": 2048, "output_len": 2048, "concurrency": 16, "num_prompts": 160}, +} +with open(p, "w") as f: + json.dump(d, f, indent=2, ensure_ascii=False) +PY + +# --- start server (unless caller manages it themselves) -------------------- +if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then + log "SKIP_MANAGE_SERVER set; assuming server already running on port ${PORT}" + if ! health_check 127.0.0.1 "$PORT" 30; then + log "ERROR: no healthy server at port ${PORT}" + exit 1 + fi +else + log "starting server via start_server.sh" + bash "${SCRIPT_DIR}/start_server.sh" >> "${LOG_DIR}/start_server.outer.log" 2>&1 +fi + +on_exit() { + local code=$? + if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then + log "stopping sglang inside ${CONTAINER_NAME} (container kept alive)" + docker exec "$CONTAINER_NAME" bash -lc 'pkill -9 -f sglang.launch_server 2>/dev/null || true' || true + fi + log "orchestrator exiting with code=${code}" + exit "$code" +} +trap on_exit EXIT + +# --- run scenarios --------------------------------------------------------- +log "===== BENCHMARK START =====" + +for scenario in "${SCENARIOS[@]}"; do + scenario="${scenario//[()]/}" + read -r concurrency input_len output_len num_prompts <<< "$scenario" + + today="$(date '+%m%d')" + # Filename convention expected by parse_results.py: + # {chip}_{engine}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl + output_file="${RAW_DIR}/${CHIP}_${ENGINE}_${today}_${concurrency}_${input_len}_${output_len}.jsonl" + # parse_results.py reads the human-readable summary from this exact path. + summary_log="${LOG_DIR}/sglang_c${concurrency}_i${input_len}_o${output_len}.log" + + log "running scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}" + log "output_file=${output_file}" + + docker exec "$CONTAINER_NAME" bash -lc " + source /root/miniconda/etc/profile.d/conda.sh + conda activate python310_torch25_cuda + python -m sglang.bench_serving \ + --backend ${BACKEND} \ + --host 127.0.0.1 \ + --port ${PORT} \ + --model ${MODEL_PATH} \ + --dataset-name random \ + --dataset-path ${DATASET_PATH} \ + --random-input-len ${input_len} \ + --random-output-len ${output_len} \ + --random-range-ratio ${RANDOM_RANGE_RATIO} \ + --num-prompts ${num_prompts} \ + --warmup-requests ${WARMUP} \ + --max-concurrency ${concurrency} \ + --request-rate ${REQUEST_RATE} \ + --output-file ${output_file} \ + --output-details \ + --seed ${SEED} + " > "$summary_log" 2>&1 || { + log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${summary_log}" + tail -60 "$summary_log" 2>/dev/null || true + continue + } + + log "finished scenario: output=${output_file}" + log "----- summary -----" + tail -40 "$summary_log" | sed 's/^/ /' +done + +log "===== BENCHMARK DONE =====" +log "parsing results..." +"${PYTHON:-python3}" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" || log "WARNING: parser failed" + +log "all results saved to ${RESULT_ROOT}" +log "report: ${RESULT_ROOT}/report.md" diff --git a/experiments/p800/qwen3_235b_p800_sglang_tp8/start_server.sh b/experiments/p800/qwen3_235b_p800_sglang_tp8/start_server.sh new file mode 100755 index 0000000..eb15845 --- /dev/null +++ b/experiments/p800/qwen3_235b_p800_sglang_tp8/start_server.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Start the SGLang server for Qwen3-235B-A22B on Kunlun P800 (TP=8, 8x XPU). +# +# Adapted from the proven qwen3-8b TP=1 launch on this image. Differences: +# - new container exposing all 8 /dev/xpu* devices (the 8b container only had xpu0) +# - --tp-size 8, XPU_VISIBLE_DEVICES=0..7 +# - context-length 8192 (8b used 4096) +# +# All XPU-specific env vars (XSGL_*, XMLIR_*) are taken verbatim from the 8b run. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# Source config.env BEFORE platform.sh so our CONTAINER_NAME default wins +# over the platform's "sglang-dsv4-flash" default (avoids name collisions +# with the dsv4 experiments, which would docker rm -f this container). +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" +LOG_DIR="${RESULT_ROOT}/logs" +mkdir -p "$LOG_DIR" + +log_init "${LOG_DIR}/start_server.log" +log "starting server for ${EXPERIMENT_NAME}" +log "model: ${MODEL_PATH}" +log "tp: ${TP} xpu: ${XPU_VISIBLE_DEVICES}" +log "port: ${PORT}" +log "container: ${CONTAINER_NAME}" +log "image: ${DOCKER_IMAGE}" + +SERVER_LOG="${LOG_DIR}/server.log" + +# --- 1. Ensure the container is running with all 8 XPU devices ------------- +ensure_container() { + if docker inspect "$CONTAINER_NAME" >/dev/null 2>&1; then + log "container ${CONTAINER_NAME} exists; ensuring it is running" + docker start "$CONTAINER_NAME" >/dev/null 2>&1 || true + else + log "creating container ${CONTAINER_NAME} from ${DOCKER_IMAGE}" + local device_args="" + local i + for i in 0 1 2 3 4 5 6 7; do + device_args="${device_args} --device /dev/xpu${i}:/dev/xpu${i}" + done + device_args="${device_args} --device /dev/xpuctrl:/dev/xpuctrl" + + docker run -d \ + --name "$CONTAINER_NAME" \ + --privileged \ + --network host \ + --ipc host \ + --security-opt label=disable \ + --shm-size 1g \ + ${device_args} \ + -v /data1:/data1 \ + -e XPU_VISIBLE_DEVICES="${XPU_VISIBLE_DEVICES}" \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e TZ=Asia/Shanghai \ + --entrypoint /bin/bash \ + "$DOCKER_IMAGE" \ + -lc "source /etc/profile >/dev/null 2>&1 || true; exec sleep infinity" + fi +} + +# --- 2. Launch sglang.launch_server inside the container ------------------- +launch_sglang() { + log "launching sglang launch_server (TP=${TP}) inside ${CONTAINER_NAME}" + # Kill any prior sglang process inside the container. + docker exec "$CONTAINER_NAME" bash -lc 'pkill -9 -f sglang.launch_server 2>/dev/null || true' || true + + docker exec -d "$CONTAINER_NAME" bash -lc " + source /root/miniconda/etc/profile.d/conda.sh + conda activate python310_torch25_cuda + export XPU_VISIBLE_DEVICES=${XPU_VISIBLE_DEVICES} + export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} + export XSGL_INTERTYPE_BFP16=1 + export XSGL_FAST_SWIGLU=1 + export XMLIR_FORCE_USE_XPU_GRAPH=1 + export XSGL_FUSE_SPLIT_NORM_ROPE_NEOX=1 + export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3000000 + export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=600 + exec python -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --host 0.0.0.0 \ + --port ${PORT} \ + --tp-size ${TP} \ + --context-length ${CONTEXT_LENGTH} \ + --max-running-requests ${MAX_RUNNING_REQUESTS} \ + --attention-backend ${ATTENTION_BACKEND} \ + --trust-remote-code \ + --disable-radix-cache \ + --disable-custom-all-reduce \ + --chunked-prefill-size -1 \ + --page-size ${PAGE_SIZE} \ + --mem-fraction-static ${MEM_FRACTION_STATIC} \ + --max-prefill-tokens ${MAX_PREFILL_TOKENS} \ + --dtype ${DTYPE} \ + --cuda-graph-max-bs ${CUDA_GRAPH_MAX_BS} \ + --cuda-graph-bs ${CUDA_GRAPH_BS} \ + --watchdog-timeout 3000000 \ + > ${SERVER_LOG} 2>&1 + " +} + +ensure_container + +# --- 1b. Patch qwen3_moe.py bug in the image ------------------------------- +# The Kunlun sglang image's qwen3_moe.py:1829 assigns to the read-only +# @property `routed_experts_weights_of_layer` instead of the private +# `_routed_experts_weights_of_layer`, raising +# AttributeError: can't set attribute 'routed_experts_weights_of_layer' +# for any qwen3 MoE model (e.g. Qwen3-235B-A22B). Qwen3-8B is dense, so the +# proven 8b run never hit this path. Fix: assign to the private attr. +# Idempotent - safe to run on every start. +patch_qwen3_moe() { + log "applying qwen3_moe.py patch in ${CONTAINER_NAME}" + docker exec "$CONTAINER_NAME" bash -lc ' + F=/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/sglang/srt/models/qwen3_moe.py + if grep -q "self\.routed_experts_weights_of_layer = LazyValue(" "$F"; then + cp "$F" "$F.bak.qwen3moe" 2>/dev/null || true + sed -i "s/self\.routed_experts_weights_of_layer = LazyValue(/self._routed_experts_weights_of_layer = LazyValue(/" "$F" + echo "PATCHED qwen3_moe.py (routed_experts_weights_of_layer -> _routed_experts_weights_of_layer)" + else + echo "qwen3_moe.py already patched" + fi + ' +} +patch_qwen3_moe + +launch_sglang + +# 235B weights (~470GB) + cuda-graph capture for 16 batch sizes take a while. +HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-1200}" +log "waiting for health on http://127.0.0.1:${PORT}/health (timeout=${HEALTH_TIMEOUT}s)" +if health_check 127.0.0.1 "$PORT" "$HEALTH_TIMEOUT"; then + log "sglang server is READY at http://127.0.0.1:${PORT}" + log "server log: ${SERVER_LOG} (or: docker logs -f ${CONTAINER_NAME})" + exit 0 +else + log "ERROR: server not healthy after ${HEALTH_TIMEOUT}s" + log "----- last 200 lines of server log -----" + tail -200 "$SERVER_LOG" 2>/dev/null || docker logs --tail 200 "$CONTAINER_NAME" 2>&1 | tail -200 + exit 1 +fi