diff --git a/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/README.md b/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/README.md new file mode 100644 index 0000000..0187601 --- /dev/null +++ b/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/README.md @@ -0,0 +1,51 @@ +# Qwen3-235B-A22B · NVIDIA RTX 6000D · SGLang TP=8 + +在 pro6000D.1（8× NVIDIA RTX 6000D，每卡 ~84GB）上用 SGLang 以 **TP=8** 部署 +`Qwen3-235B-A22B`，跑 **输入 2048 / 输出 2048 / 并发 16 / 160 请求** 基准， +与 `experiments/p800/qwen3_235b_p800_sglang_tp8`（昆仑 P800）做**跨机器对比**。 + +## 与 p800.2 实验的关系 + +工作负载完全相同（isl=osl=2048, c=16, n=160, seed=1, request-rate=10000），仅部署 +按平台适配： + +| 项 | p800.2 (Kunlun P800 XPU) | pro6000D.1 (NVIDIA RTX 6000D) | +|---|---|---| +| 镜像 | 昆仑 sglang 镜像 (pd-disagg-0510) | `sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1` | +| 设备 | 8× XPU (`/dev/xpu*`, `--device`) | 8× GPU (`--gpus all`) | +| attention backend | `--attention-backend kunlun` | sglang 默认 (flashinfer) | +| dtype | float16 (+ XSGL_INTERTYPE_BFP16) | auto (bf16，NVIDIA 原生) | +| qwen3_moe 补丁 | 需要（镜像 bug） | **不需要**（标准 sglang） | +| XPU 环境变量 | XSGL_* 一堆 | 无 | +| TP / mem-fraction / ctx / max-running | 8 / 0.9 / 8192 / 16 | 8 / 0.9 / 8192 / 16（一致） | + +> dtype 差异说明：p800.2 用 fp16（昆仑原生），本机用 bf16（NVIDIA 原生）。两者在 +> 各自平台上是自然精度；fp16/bf16 计算吞吐基本相同，对比有效。 + +## 设计 + +- `start_server.sh`：`docker run --rm --gpus all` 起 sglang 容器，标准 qwen3_moe + 启动参数（无昆仑补丁/XPU env），nohup 后台 + 健康检查。挂载 `/data:/data` 使模型、 + 结果目录、数据集在容器内外同路径。 +- bench 客户端 `sglang.bench_serving` 通过 `docker exec` 跑在服务容器内。 +- `parse_results.py` 复用自 dsv4_p800_sglang（与 p800.2 实验同一份）。 + +显存：235B bf16 ~470GB，TP=8 每卡 ~59GB；每卡 84GB，0.9 静态占比留 ~16GB/卡 给 +KV+激活+cuda graph，2k+2k 上下文 KV 极小，充裕。 + +## 用法 + +```bash +cd /data/yy/sskj/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8 +bash run_bench.sh # 一键：起服务 + 基准 + 解析 +# 或分步： +bash start_server.sh +SKIP_MANAGE_SERVER=1 bash run_bench.sh +``` + +结果在 `results//`：`report.md`、`results.json`、`raw_outputs/*.jsonl`、`logs/`。 + +## 输出指标 + +每个 scenario：Success/Failed、Req/s、Input/Output/Total tok/s、TTFT/TPOT/E2E +（mean/p50/p90/p99）。与 p800.2 的对比见对话或自行用两边 `results.json` 汇总。 diff --git a/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/config.env b/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/config.env new file mode 100644 index 0000000..d3e094d --- /dev/null +++ b/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/config.env @@ -0,0 +1,50 @@ +# Experiment: Qwen3-235B-A22B on NVIDIA RTX 6000D (8x 84GB), SGLang, TP=8. +# Cross-machine counterpart of experiments/p800/qwen3_235b_p800_sglang_tp8 +# (Kunlun P800). Same workload: isl=2048 osl=2048 concurrency=16 num_prompts=160. +# +# NVIDIA deployment uses the standard sglang qwen3_moe path (no Kunlun +# qwen3_moe.py patch, no XPU env, no --attention-backend kunlun). dtype is +# left to sglang auto (bf16) - NVIDIA native; p800.2 ran float16 (Kunlun). + +EXPERIMENT="${EXPERIMENT:-qwen3_235b_pro6000_sglang_tp8}" +MODEL_NAME="${MODEL_NAME:-Qwen3-235B-A22B}" +MODEL_PATH="${MODEL_PATH:-/data/6000D/Qwen3-235B-A22B}" +SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-/data/6000D/Qwen3-235B-A22B}" +BACKEND="${BACKEND:-sglang}" +ENGINE="${ENGINE:-sglang}" +DATASET="${DATASET:-random}" +DATASET_PATH="${DATASET_PATH:-/data/yy/sskj/dataset/ShareGPT_V3_unfiltered_cleaned_split.json}" + +# Serving port + container. +PORT="${PORT:-30010}" +CONTAINER_NAME="${CONTAINER_NAME:-qwen3_235b_pro6000_sglang_tp8}" + +# Docker image: sm120-capable SGLang (built/verified on this RTX 6000D box), +# sglang 0.5.15.post1 supports qwen3_moe. Falls back to nightly cu13 if needed. +DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}" +# Persistent JIT cache (flashinfer/sglang) shared with other pro6000 experiments. +CACHE_DIR="${CACHE_DIR:-/data/yy/sskj/sglang_sm120_cache}" + +# Hardware: 8x NVIDIA RTX 6000D, TP=8. +TP="${TP:-8}" +CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" + +# SGLang launch params. Kept standard (NVIDIA natural config); only TP, +# mem-fraction, context-length, max-running-requests are pinned to match the +# p800.2 run for comparability. +CONTEXT_LENGTH="${CONTEXT_LENGTH:-8192}" +MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-16}" +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.9}" +# dtype: auto (bf16). Set DTYPE=float16 to match p800.2 exactly (risk: fp16 +# overflow on a bf16 model on NVIDIA). +DTYPE="${DTYPE:-auto}" + +# Bench scenario: "concurrency input_len output_len num_prompts". +SCENARIOS=( + "16 2048 2048 160" +) + +WARMUP="${WARMUP:-16}" +REQUEST_RATE="${REQUEST_RATE:-10000}" +SEED="${SEED:-1}" +RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1}" diff --git a/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/parse_results.py b/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/parse_results.py new file mode 100755 index 0000000..341f69e --- /dev/null +++ b/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/parse_results.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""Parse P800 SGLang benchmark outputs and produce results.json + report.md. + +Usage: + python3 parse_results.py +""" + +import json +import os +import re +import sys +from pathlib import Path +from collections import OrderedDict + +METRIC_PATTERNS = OrderedDict( + [ + ("successful_requests", [r"Successful requests:\s+(\d+)"]), + ("benchmark_duration_s", [r"Benchmark duration \(s\):\s+([\d.]+)"]), + ("request_throughput", [r"Request throughput \(req/s\):\s+([\d.]+)"]), + ("input_token_throughput", [r"Input token throughput \(tok/s\):\s+([\d.]+)"]), + ("output_token_throughput", [r"Output token throughput \(tok/s\):\s+([\d.]+)"]), + ("total_token_throughput", [r"Total token throughput \(tok/s\):\s+([\d.]+)"]), + ("ttft_mean", [r"Mean TTFT \(ms\):\s+([\d.]+)"]), + ("ttft_p50", [r"Median TTFT \(ms\):\s+([\d.]+)", r"P50 TTFT \(ms\):\s+([\d.]+)"]), + ("ttft_p90", [r"P90 TTFT \(ms\):\s+([\d.]+)"]), + ("ttft_p99", [r"P99 TTFT \(ms\):\s+([\d.]+)"]), + ("tpot_mean", [r"Mean TPOT \(ms\):\s+([\d.]+)"]), + ("tpot_p50", [r"Median TPOT \(ms\):\s+([\d.]+)", r"P50 TPOT \(ms\):\s+([\d.]+)"]), + ("tpot_p90", [r"P90 TPOT \(ms\):\s+([\d.]+)"]), + ("tpot_p99", [r"P99 TPOT \(ms\):\s+([\d.]+)"]), + ("e2e_mean", [r"Mean E2E \(ms\):\s+([\d.]+)"]), + ("e2e_p50", [r"Median E2E \(ms\):\s+([\d.]+)", r"P50 E2E \(ms\):\s+([\d.]+)"]), + ("e2e_p90", [r"P90 E2E \(ms\):\s+([\d.]+)"]), + ("e2e_p99", [r"P99 E2E \(ms\):\s+([\d.]+)"]), + ] +) + + +def find_float(text: str, patterns: list[str]) -> float | None: + for pat in patterns: + m = re.search(pat, text) + if m: + try: + return float(m.group(1)) + except ValueError: + return None + return None + + +def parse_summary_log(log_text: str) -> dict: + result = {} + for key, patterns in METRIC_PATTERNS.items(): + result[key] = find_float(log_text, patterns) + return result + + +def percentile(values: list[float], p: float) -> float: + if not values: + return 0.0 + sorted_values = sorted(values) + k = (len(sorted_values) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(sorted_values) - 1) + if f == c: + return sorted_values[f] + return sorted_values[f] * (c - k) + sorted_values[c] * (k - f) + + +def parse_jsonl(jsonl_path: Path) -> tuple[list[dict], dict]: + """Parse sglang.bench_serving --output-file output. + + Newer sglang writes one aggregate JSON object. Older versions write one + JSON object per request. Returns (raw_requests, aggregates). + """ + text = jsonl_path.read_text(encoding="utf-8", errors="replace").strip() + if not text: + return [], {} + + # Try single aggregate JSON object first. + try: + data = json.loads(text) + if isinstance(data, dict) and "mean_e2e_latency_ms" in data: + aggregates = { + "success": data.get("total_output_tokens", 0) > 0 and 1 or 0, + "failed": 0, + "input_tokens": data.get("total_input_tokens", 0), + "output_tokens": data.get("total_output_tokens", 0), + "latencies": { + "e2e_ms": { + "mean": data.get("mean_e2e_latency_ms"), + "p50": data.get("median_e2e_latency_ms"), + "p90": data.get("p90_e2e_latency_ms"), + "p95": None, + "p99": data.get("p99_e2e_latency_ms"), + }, + "ttft_ms": { + "mean": data.get("mean_ttft_ms"), + "p50": data.get("median_ttft_ms"), + "p90": None, + "p95": None, + "p99": data.get("p99_ttft_ms"), + }, + "tpot_ms": { + "mean": data.get("mean_tpot_ms"), + "p50": data.get("median_tpot_ms"), + "p90": None, + "p95": None, + "p99": data.get("p99_tpot_ms"), + }, + "itl_ms": { + "mean": data.get("mean_itl_ms"), + "p50": data.get("median_itl_ms"), + "p90": None, + "p95": data.get("p95_itl_ms"), + "p99": data.get("p99_itl_ms"), + }, + }, + } + return [data], aggregates + except json.JSONDecodeError: + pass + + # Fall back to JSONL per-request parsing. + raw_requests = [] + ttfts = [] + tpots = [] + itls = [] + e2es = [] + input_tokens = [] + output_tokens = [] + success = 0 + failed = 0 + + for line in text.splitlines(): + line = line.strip() + if not line: + continue + try: + req = json.loads(line) + except json.JSONDecodeError: + continue + + raw_requests.append(req) + + ttft = req.get("ttft") or req.get("ttft_ms") or 0 + tpot = req.get("tpot") or req.get("tpot_ms") or 0 + itl = req.get("inter_token_latency") or req.get("itl") or req.get("itl_ms") or 0 + e2e = req.get("e2e_latency") or req.get("e2e") or req.get("e2e_ms") or 0 + in_tok = req.get("input_tokens") or req.get("prompt_tokens") or 0 + out_tok = req.get("output_tokens") or req.get("completion_tokens") or 0 + + if ttft: + ttfts.append(float(ttft)) + if tpot: + tpots.append(float(tpot)) + if itl: + itls.append(float(itl)) + if e2e: + e2es.append(float(e2e)) + if in_tok: + input_tokens.append(int(in_tok)) + if out_tok: + output_tokens.append(int(out_tok)) + + if req.get("success", True): + success += 1 + else: + failed += 1 + + def latency_bucket(values: list[float]) -> dict: + if not values: + return {"mean": None, "p50": None, "p90": None, "p95": None, "p99": None} + return { + "mean": round(sum(values) / len(values), 2), + "p50": round(percentile(values, 50), 2), + "p90": round(percentile(values, 90), 2), + "p95": round(percentile(values, 95), 2), + "p99": round(percentile(values, 99), 2), + } + + aggregates = { + "success": success, + "failed": failed, + "input_tokens": sum(input_tokens), + "output_tokens": sum(output_tokens), + "latencies": { + "e2e_ms": latency_bucket(e2es), + "ttft_ms": latency_bucket(ttfts), + "tpot_ms": latency_bucket(tpots), + "itl_ms": latency_bucket(itls), + }, + } + + return raw_requests, aggregates + + +def parse_scenario(result_root: Path, raw_file: Path) -> dict | None: + """Parse one raw output file into a scenario dict.""" + # Filename: {chip}_{engine}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl + parts = raw_file.stem.split("_") + if len(parts) < 6: + return None + try: + concurrency = int(parts[-3]) + input_len = int(parts[-2]) + output_len = int(parts[-1]) + except ValueError: + return None + + log_file = result_root / "logs" / f"sglang_c{concurrency}_i{input_len}_o{output_len}.log" + summary = {} + if log_file.exists(): + summary = parse_summary_log(log_file.read_text(encoding="utf-8", errors="replace")) + + raw_requests, aggregates = parse_jsonl(raw_file) + + # For newer sglang aggregate JSON, success/failed are not present in the + # raw output file. Override them from the human-readable summary log when + # it is available. + if summary.get("successful_requests") is not None: + aggregates["success"] = int(summary["successful_requests"]) + # The summary log only reports successes; assume failures are zero + # unless the aggregate JSON already provided a non-zero failed count. + if not aggregates.get("failed"): + aggregates["failed"] = 0 + + scenario = { + "name": f"c{concurrency}_i{input_len}_o{output_len}", + "concurrency": concurrency, + "input_len": input_len, + "output_len": output_len, + "success": aggregates["success"], + "failed": aggregates["failed"], + "duration_s": summary.get("benchmark_duration_s"), + "request_throughput": summary.get("request_throughput"), + "input_token_throughput": summary.get("input_token_throughput"), + "output_token_throughput": summary.get("output_token_throughput"), + "total_token_throughput": summary.get("total_token_throughput"), + "accept_length": None, + "latencies": aggregates["latencies"], + "raw_requests": raw_requests[:100] if len(raw_requests) <= 100 else None, + } + + return scenario + + +def main() -> None: + result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") + raw_dir = result_root / "raw_outputs" + json_path = result_root / "results.json" + report_path = result_root / "report.md" + + if not json_path.exists(): + raise SystemExit(f"metadata results.json not found: {json_path}") + + with open(json_path, "r", encoding="utf-8") as f: + data = json.load(f) + + scenarios = [] + if raw_dir.exists(): + for raw_file in sorted(raw_dir.glob("*.jsonl")): + scenario = parse_scenario(result_root, raw_file) + if scenario: + scenarios.append(scenario) + + data["scenarios"] = scenarios + + with open(json_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + # Generate report.md + with open(report_path, "w", encoding="utf-8") as f: + meta = data["metadata"] + f.write(f"# Benchmark Report: {meta['experiment']}\n\n") + f.write("## Metadata\n\n") + f.write(f"- **Run ID**: {meta['run_id']}\n") + f.write(f"- **Timestamp**: {meta['timestamp']}\n") + f.write(f"- **Chip/Accelerator**: {meta.get('accelerator', '')} / {meta.get('chip', '')}\n") + f.write(f"- **Engine/Backend**: {meta.get('engine', '')} / {meta.get('backend', '')}\n") + f.write(f"- **Hardware**: {meta.get('hardware', '')}\n") + f.write(f"- **Model**: {meta['model']}\n") + f.write(f"- **Git Commit**: {meta.get('git_commit', 'unknown')}\n") + f.write("\n## Results\n\n") + f.write( + "| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | " + "TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 |\n" + ) + f.write( + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n" + ) + for s in scenarios: + lat = s["latencies"] + f.write( + f"| {s['name']} " + f"| {s['concurrency']} " + f"| {s['input_len']}/{s['output_len']} " + f"| {s['success']} " + f"| {s['failed']} " + f"| {s.get('request_throughput') or ''} " + f"| {s.get('output_token_throughput') or ''} " + f"| {lat['ttft_ms']['p50'] or ''} " + f"| {lat['ttft_ms']['p99'] or ''} " + f"| {lat['tpot_ms']['p50'] or ''} " + f"| {lat['tpot_ms']['p99'] or ''} " + f"| {lat['e2e_ms']['p99'] or ''} |\n" + ) + + print(f"Updated {json_path}") + print(f"Wrote {report_path}") + + +if __name__ == "__main__": + main() diff --git a/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/results/run_20260721_231606/report.md b/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/results/run_20260721_231606/report.md new file mode 100644 index 0000000..1f949f0 --- /dev/null +++ b/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/results/run_20260721_231606/report.md @@ -0,0 +1,17 @@ +# Benchmark Report: qwen3_235b_pro6000_sglang_tp8 + +## Metadata + +- **Run ID**: run_20260721_231606 +- **Timestamp**: 2026-07-21T23:16:07+08:00 +- **Chip/Accelerator**: NVIDIA RTX 6000D / nvidia_rtx6000d +- **Engine/Backend**: sglang / sglang +- **Hardware**: 8x NVIDIA RTX 6000D 84GB +- **Model**: /data/6000D/Qwen3-235B-A22B +- **Git Commit**: 5006de6 + +## Results + +| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c16_i2048_o2048 | 16 | 2048/2048 | 160 | 0 | 0.19 | 398.19 | 4881.394433992682 | 77666.54609258809 | 33.548958103309424 | 48.34612468822903 | 150572.5178761262 | diff --git a/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/run_bench.sh b/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/run_bench.sh new file mode 100755 index 0000000..7524e6e --- /dev/null +++ b/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/run_bench.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Orchestrator for the Qwen3-235B-A22B TP=8 benchmark on NVIDIA RTX 6000D. +# Cross-machine counterpart of experiments/p800/qwen3_235b_p800_sglang_tp8/run_bench.sh. +# +# Flow: write metadata -> start server (start_server.sh) -> bench_serving +# (isl=2048 osl=2048 c=16 n=160, inside the container) -> parse -> report.md. +# The bench client runs via `docker exec` into the running server container; +# /data is mounted same-path so results + dataset are reachable inside. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" +RAW_DIR="${RESULT_ROOT}/raw_outputs" +LOG_DIR="${RESULT_ROOT}/logs" +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "$RAW_DIR" "$LOG_DIR" + +ensure_result_root "$RESULT_ROOT" +log_init "${LOG_DIR}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "result_root=${RESULT_ROOT}" +log "platform=${PLATFORM} chip=${CHIP} hardware=${HARDWARE}" +log "model=${MODEL_PATH} tp=${TP} backend=${BACKEND} image=${DOCKER_IMAGE}" +log "scenarios=${SCENARIOS[*]}" + +# --- metadata -------------------------------------------------------------- +METADATA_JSON="${RESULT_ROOT}/results.json" +write_metadata_json \ + "$METADATA_JSON" \ + "$EXPERIMENT_NAME" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "$BACKEND" \ + "$ENGINE" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "" \ + "Qwen3-235B-A22B SGLang TP=8 benchmark on NVIDIA RTX 6000D (isl=osl=2048, c=16) - cross-machine vs p800.2" + +"${PYTHON:-python3}" - "$METADATA_JSON" "$TP" "$CONTEXT_LENGTH" "$PORT" "$CONTAINER_NAME" "$DOCKER_IMAGE" "$MEM_FRACTION_STATIC" "$DTYPE" <<'PY' +import json, sys +p, tp, ctx, port, cont, img, memfrac, dtype = sys.argv[1:9] +with open(p) as f: + d = json.load(f) +d["config"] = { + "tp": int(tp), + "context_length": int(ctx), + "port": int(port), + "container": cont, + "docker_image": img, + "dtype": dtype, + "mem_fraction_static": float(memfrac), + "max_running_requests": 16, + "benchmark": {"input_len": 2048, "output_len": 2048, "concurrency": 16, "num_prompts": 160}, +} +with open(p, "w") as f: + json.dump(d, f, indent=2, ensure_ascii=False) +PY + +# --- start server (unless caller manages it themselves) -------------------- +if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then + log "SKIP_MANAGE_SERVER set; assuming server already running on port ${PORT}" + if ! health_check 127.0.0.1 "$PORT" 30; then + log "ERROR: no healthy server at port ${PORT}" + exit 1 + fi +else + log "starting server via start_server.sh" + bash "${SCRIPT_DIR}/start_server.sh" >> "${LOG_DIR}/start_server.outer.log" 2>&1 +fi + +stop_server() { + if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then + log "stopping server (kill docker run PID -> --rm removes container)" + if [[ -f "${RUNTIME_BASE}/${EXPERIMENT_NAME}.pid" ]]; then + local pid + pid="$(cat "${RUNTIME_BASE}/${EXPERIMENT_NAME}.pid")" + kill "$pid" 2>/dev/null || true + sleep 3 + kill -9 "$pid" 2>/dev/null || true + fi + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + fi +} +on_exit() { + local code=$? + stop_server + log "orchestrator exiting with code=${code}" + exit "$code" +} +trap on_exit EXIT + +# --- run scenarios --------------------------------------------------------- +log "===== BENCHMARK START =====" + +for scenario in "${SCENARIOS[@]}"; do + scenario="${scenario//[()]/}" + read -r concurrency input_len output_len num_prompts <<< "$scenario" + + today="$(date '+%m%d')" + # Filename convention expected by parse_results.py: + # {chip}_{engine}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl + output_file="${RAW_DIR}/${CHIP}_${ENGINE}_${today}_${concurrency}_${input_len}_${output_len}.jsonl" + summary_log="${LOG_DIR}/sglang_c${concurrency}_i${input_len}_o${output_len}.log" + + log "running scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}" + log "output_file=${output_file}" + + docker exec "$CONTAINER_NAME" python3 -m sglang.bench_serving \ + --backend "${BACKEND}" \ + --host 127.0.0.1 \ + --port "${PORT}" \ + --model "${MODEL_PATH}" \ + --dataset-name random \ + --dataset-path "${DATASET_PATH}" \ + --random-input-len "${input_len}" \ + --random-output-len "${output_len}" \ + --random-range-ratio "${RANDOM_RANGE_RATIO}" \ + --num-prompts "${num_prompts}" \ + --warmup-requests "${WARMUP}" \ + --max-concurrency "${concurrency}" \ + --request-rate "${REQUEST_RATE}" \ + --output-file "${output_file}" \ + --output-details \ + --seed "${SEED}" \ + > "$summary_log" 2>&1 || { + log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${summary_log}" + tail -60 "$summary_log" 2>/dev/null || true + continue + } + + log "finished scenario: output=${output_file}" + log "----- summary -----" + tail -40 "$summary_log" | sed 's/^/ /' +done + +log "===== BENCHMARK DONE =====" +log "parsing results..." +"${PYTHON:-python3}" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" || log "WARNING: parser failed" + +log "all results saved to ${RESULT_ROOT}" +log "report: ${RESULT_ROOT}/report.md" diff --git a/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/start_server.sh b/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/start_server.sh new file mode 100755 index 0000000..613d8bd --- /dev/null +++ b/experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/start_server.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Start the SGLang server for Qwen3-235B-A22B on NVIDIA RTX 6000D (TP=8, 8 GPUs). +# +# NVIDIA counterpart of experiments/p800/qwen3_235b_p800_sglang_tp8/start_server.sh. +# Uses the standard sglang qwen3_moe path (no Kunlun patch, no XPU env). +# Launch pattern mirrors pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_docker.sh. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# Source config before platform so our CONTAINER_NAME/PORT defaults win. +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +LOG_DIR="${RESULT_ROOT}/logs" +mkdir -p "$LOG_DIR" "${RUNTIME_BASE}/tmp" "$CACHE_DIR" + +log_init "${LOG_DIR}/start_server.log" +log "starting server for ${EXPERIMENT_NAME} (NVIDIA, TP=${TP})" +log "model: ${MODEL_PATH}" +log "image: ${DOCKER_IMAGE}" +log "port: ${PORT} container: ${CONTAINER_NAME}" + +SERVER_LOG="${LOG_DIR}/server.log" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT_NAME}.pid" +rm -f "$PID_FILE" + +# Remove any stale container with the same name. +docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + +# Standard sglang launch for qwen3_moe on NVIDIA (bf16 auto, flashinfer attn). +# Mount /data so model + experiment results dir + dataset are all at the same +# path inside the container (bench client writes results there directly). +DTYPE_ARGS=() +if [[ "${DTYPE}" != "auto" ]]; then + DTYPE_ARGS=(--dtype "${DTYPE}") +fi + +nohup docker run --rm \ + --name "$CONTAINER_NAME" \ + --gpus all \ + --privileged \ + --ipc=host \ + --network host \ + --ulimit memlock=-1 \ + --ulimit stack=67108864 \ + --entrypoint python3 \ + -v /data:/data \ + -v "${CACHE_DIR}:/root/.cache" \ + -v "${RUNTIME_BASE}/tmp:/tmp" \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ + "$DOCKER_IMAGE" \ + -m sglang.launch_server \ + --model-path "${MODEL_PATH}" \ + --trust-remote-code \ + --tp-size "${TP}" \ + --mem-fraction-static "${MEM_FRACTION_STATIC}" \ + --context-length "${CONTEXT_LENGTH}" \ + --max-running-requests "${MAX_RUNNING_REQUESTS}" \ + --host 0.0.0.0 \ + --port "${PORT}" \ + "${DTYPE_ARGS[@]}" \ + > "$SERVER_LOG" 2>&1 & + +PID=$! +echo "$PID" > "$PID_FILE" +log "PID: ${PID}" +HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-1200}" +log "waiting for health on http://127.0.0.1:${PORT}/health (timeout=${HEALTH_TIMEOUT}s)" + +if health_check 127.0.0.1 "$PORT" "$HEALTH_TIMEOUT"; then + log "sglang server is READY at http://127.0.0.1:${PORT}" + log "server log: ${SERVER_LOG} (container: ${CONTAINER_NAME})" + exit 0 +else + log "ERROR: server not healthy after ${HEALTH_TIMEOUT}s" + log "----- last 200 lines of server log -----" + tail -200 "$SERVER_LOG" 2>/dev/null || docker logs --tail 200 "$CONTAINER_NAME" 2>&1 | tail -200 + exit 1 +fi