From be94b8e7efbad9d4588ad83a4ac2fdd3ab6395e3 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Wed, 8 Jul 2026 10:30:05 +0000 Subject: [PATCH] Add P800 256k/4k long-context probe and baseline w8a8_int8 server mode - Add experiments/dsv4_p800_256k_4k_probe for testing 256k input + 4k output on DeepSeek-V4-Flash-INT8 with Kunlun P800. - Add SGLANG_EXTRA_LAUNCH_ARGS support to scripts/common/server_docker.sh so probe can inject --context-length 270000. - Add w8a8_int8_baseline server mode without EAGLE speculative decoding, matching the long-context P800 tuning. - Switch dsv4_p800_sglang and the new probe to use w8a8_int8_baseline by default; original tuned mode remains available via SERVER_MODE=w8a8_int8. --- experiments/dsv4_p800_256k_4k_probe/README.md | 63 ++++ .../dsv4_p800_256k_4k_probe/config.env | 32 ++ .../dsv4_p800_256k_4k_probe/parse_results.py | 313 ++++++++++++++++++ .../dsv4_p800_256k_4k_probe/run_bench.sh | 156 +++++++++ .../dsv4_p800_256k_4k_probe/start_server.sh | 19 ++ experiments/dsv4_p800_sglang/README.md | 4 + experiments/dsv4_p800_sglang/config.env | 7 +- scripts/common/server_docker.sh | 11 + 8 files changed, 602 insertions(+), 3 deletions(-) create mode 100644 experiments/dsv4_p800_256k_4k_probe/README.md create mode 100644 experiments/dsv4_p800_256k_4k_probe/config.env create mode 100755 experiments/dsv4_p800_256k_4k_probe/parse_results.py create mode 100755 experiments/dsv4_p800_256k_4k_probe/run_bench.sh create mode 100755 experiments/dsv4_p800_256k_4k_probe/start_server.sh diff --git a/experiments/dsv4_p800_256k_4k_probe/README.md b/experiments/dsv4_p800_256k_4k_probe/README.md new file mode 100644 index 0000000..ac8c6b6 --- /dev/null +++ b/experiments/dsv4_p800_256k_4k_probe/README.md @@ -0,0 +1,63 @@ +# DSV4 P800 256k-Input / 4k-Output Probe + +Kunlun P800 XPU + SGLang + `DeepSeek-V4-Flash-INT8` long-context OOM probe. + +## What it does + +Sends 4 prompts with **262144 input tokens** and requests **4096 output tokens** +at **concurrency=2**, then checks whether the server completes the scenario +without OOM or context-length errors. + +## Quick Start + +### Against an already-running server (recommended on shared P800 machines) + +```bash +PLATFORM=kunlun_p800 \ +SKIP_MANAGE_SERVER=1 \ +PORT=30000 \ +bash experiments/dsv4_p800_256k_4k_probe/run_bench.sh +``` + +### Standalone (starts its own Docker container) + +```bash +PLATFORM=kunlun_p800 \ +bash experiments/dsv4_p800_256k_4k_probe/run_bench.sh +``` + +This will start a fresh `sglang-dsv4-flash` container with the baseline +`w8a8_int8` launch args (no EAGLE speculative decoding), plus +`--context-length 270000 --max-running-requests 4`, and tear it down on exit. + +To use the tuned config with EAGLE speculative decoding instead: + +```bash +SERVER_MODE=w8a8_int8 PLATFORM=kunlun_p800 \ +bash experiments/dsv4_p800_256k_4k_probe/run_bench.sh +``` + +## Configuration + +Edit `config.env` or override via environment variables: + +```bash +# Use a different port or model path +PORT=30001 \ +MODEL_PATH=/data1/models/DeepSeek-V4-Flash-INT8 \ +PLATFORM=kunlun_p800 \ +bash experiments/dsv4_p800_256k_4k_probe/run_bench.sh +``` + +## Files + +| File | Purpose | +|---|---| +| `config.env` | Experiment-level configuration (model, port, scenario) | +| `start_server.sh` | Start the P800 SGLang Docker container with 256k context args | +| `run_bench.sh` | Orchestrator: server → warmup → probe scenario → stop server | +| `parse_results.py` | Parse logs and generate `results.json` + `report.md` | + +## Platform + +This experiment targets `platforms/kunlun_p800.env`. diff --git a/experiments/dsv4_p800_256k_4k_probe/config.env b/experiments/dsv4_p800_256k_4k_probe/config.env new file mode 100644 index 0000000..de8af0a --- /dev/null +++ b/experiments/dsv4_p800_256k_4k_probe/config.env @@ -0,0 +1,32 @@ +# OOM probe for 256k input + 4k output on DeepSeek-V4-Flash-INT8 (Kunlun P800). +# Tests whether the P800 SGLang deployment can serve this context length +# without OOM at concurrency=2 before adding it to the full long-context matrix. + +EXPERIMENT="${EXPERIMENT:-dsv4_p800_256k_4k_probe}" +MODEL_NAME="${MODEL_NAME:-DeepSeek-V4-Flash-INT8}" +MODEL_PATH="${MODEL_PATH:-/data1/models/DeepSeek-V4-Flash-INT8}" +SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-deepseek-v4-flash-int8}" +PORT="${PORT:-30000}" +BACKEND="${BACKEND:-sglang}" +ENGINE="${ENGINE:-sglang-xpu}" +DATASET="${DATASET:-random}" +DATASET_PATH="${DATASET_PATH:-/workspace/dummy_sharegpt.json}" + +# Server launch mode for P800 SGLang. +# - "w8a8_int8" : default tuned config with EAGLE speculative decoding +# - "w8a8_int8_baseline": baseline config without speculative decoding, +# matching the long-context P800 tuning. +SERVER_MODE="${SERVER_MODE:-w8a8_int8_baseline}" + +# Extra SGLang launch arguments for the 256k-context probe. +# 262144 input + 4096 output + padding -> use 270000. +SGLANG_EXTRA_LAUNCH_ARGS="${SGLANG_EXTRA_LAUNCH_ARGS:---context-length 270000 --max-running-requests 4}" + +# Scenario list: "concurrency input_len output_len" (one per line). +SCENARIOS=( + "2 262144 4096" +) + +# Number of prompts for the probe scenario. +NUM_PROMPTS="${NUM_PROMPTS:-4}" +WARMUP="${WARMUP:-0}" diff --git a/experiments/dsv4_p800_256k_4k_probe/parse_results.py b/experiments/dsv4_p800_256k_4k_probe/parse_results.py new file mode 100755 index 0000000..341f69e --- /dev/null +++ b/experiments/dsv4_p800_256k_4k_probe/parse_results.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""Parse P800 SGLang benchmark outputs and produce results.json + report.md. + +Usage: + python3 parse_results.py +""" + +import json +import os +import re +import sys +from pathlib import Path +from collections import OrderedDict + +METRIC_PATTERNS = OrderedDict( + [ + ("successful_requests", [r"Successful requests:\s+(\d+)"]), + ("benchmark_duration_s", [r"Benchmark duration \(s\):\s+([\d.]+)"]), + ("request_throughput", [r"Request throughput \(req/s\):\s+([\d.]+)"]), + ("input_token_throughput", [r"Input token throughput \(tok/s\):\s+([\d.]+)"]), + ("output_token_throughput", [r"Output token throughput \(tok/s\):\s+([\d.]+)"]), + ("total_token_throughput", [r"Total token throughput \(tok/s\):\s+([\d.]+)"]), + ("ttft_mean", [r"Mean TTFT \(ms\):\s+([\d.]+)"]), + ("ttft_p50", [r"Median TTFT \(ms\):\s+([\d.]+)", r"P50 TTFT \(ms\):\s+([\d.]+)"]), + ("ttft_p90", [r"P90 TTFT \(ms\):\s+([\d.]+)"]), + ("ttft_p99", [r"P99 TTFT \(ms\):\s+([\d.]+)"]), + ("tpot_mean", [r"Mean TPOT \(ms\):\s+([\d.]+)"]), + ("tpot_p50", [r"Median TPOT \(ms\):\s+([\d.]+)", r"P50 TPOT \(ms\):\s+([\d.]+)"]), + ("tpot_p90", [r"P90 TPOT \(ms\):\s+([\d.]+)"]), + ("tpot_p99", [r"P99 TPOT \(ms\):\s+([\d.]+)"]), + ("e2e_mean", [r"Mean E2E \(ms\):\s+([\d.]+)"]), + ("e2e_p50", [r"Median E2E \(ms\):\s+([\d.]+)", r"P50 E2E \(ms\):\s+([\d.]+)"]), + ("e2e_p90", [r"P90 E2E \(ms\):\s+([\d.]+)"]), + ("e2e_p99", [r"P99 E2E \(ms\):\s+([\d.]+)"]), + ] +) + + +def find_float(text: str, patterns: list[str]) -> float | None: + for pat in patterns: + m = re.search(pat, text) + if m: + try: + return float(m.group(1)) + except ValueError: + return None + return None + + +def parse_summary_log(log_text: str) -> dict: + result = {} + for key, patterns in METRIC_PATTERNS.items(): + result[key] = find_float(log_text, patterns) + return result + + +def percentile(values: list[float], p: float) -> float: + if not values: + return 0.0 + sorted_values = sorted(values) + k = (len(sorted_values) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(sorted_values) - 1) + if f == c: + return sorted_values[f] + return sorted_values[f] * (c - k) + sorted_values[c] * (k - f) + + +def parse_jsonl(jsonl_path: Path) -> tuple[list[dict], dict]: + """Parse sglang.bench_serving --output-file output. + + Newer sglang writes one aggregate JSON object. Older versions write one + JSON object per request. Returns (raw_requests, aggregates). + """ + text = jsonl_path.read_text(encoding="utf-8", errors="replace").strip() + if not text: + return [], {} + + # Try single aggregate JSON object first. + try: + data = json.loads(text) + if isinstance(data, dict) and "mean_e2e_latency_ms" in data: + aggregates = { + "success": data.get("total_output_tokens", 0) > 0 and 1 or 0, + "failed": 0, + "input_tokens": data.get("total_input_tokens", 0), + "output_tokens": data.get("total_output_tokens", 0), + "latencies": { + "e2e_ms": { + "mean": data.get("mean_e2e_latency_ms"), + "p50": data.get("median_e2e_latency_ms"), + "p90": data.get("p90_e2e_latency_ms"), + "p95": None, + "p99": data.get("p99_e2e_latency_ms"), + }, + "ttft_ms": { + "mean": data.get("mean_ttft_ms"), + "p50": data.get("median_ttft_ms"), + "p90": None, + "p95": None, + "p99": data.get("p99_ttft_ms"), + }, + "tpot_ms": { + "mean": data.get("mean_tpot_ms"), + "p50": data.get("median_tpot_ms"), + "p90": None, + "p95": None, + "p99": data.get("p99_tpot_ms"), + }, + "itl_ms": { + "mean": data.get("mean_itl_ms"), + "p50": data.get("median_itl_ms"), + "p90": None, + "p95": data.get("p95_itl_ms"), + "p99": data.get("p99_itl_ms"), + }, + }, + } + return [data], aggregates + except json.JSONDecodeError: + pass + + # Fall back to JSONL per-request parsing. + raw_requests = [] + ttfts = [] + tpots = [] + itls = [] + e2es = [] + input_tokens = [] + output_tokens = [] + success = 0 + failed = 0 + + for line in text.splitlines(): + line = line.strip() + if not line: + continue + try: + req = json.loads(line) + except json.JSONDecodeError: + continue + + raw_requests.append(req) + + ttft = req.get("ttft") or req.get("ttft_ms") or 0 + tpot = req.get("tpot") or req.get("tpot_ms") or 0 + itl = req.get("inter_token_latency") or req.get("itl") or req.get("itl_ms") or 0 + e2e = req.get("e2e_latency") or req.get("e2e") or req.get("e2e_ms") or 0 + in_tok = req.get("input_tokens") or req.get("prompt_tokens") or 0 + out_tok = req.get("output_tokens") or req.get("completion_tokens") or 0 + + if ttft: + ttfts.append(float(ttft)) + if tpot: + tpots.append(float(tpot)) + if itl: + itls.append(float(itl)) + if e2e: + e2es.append(float(e2e)) + if in_tok: + input_tokens.append(int(in_tok)) + if out_tok: + output_tokens.append(int(out_tok)) + + if req.get("success", True): + success += 1 + else: + failed += 1 + + def latency_bucket(values: list[float]) -> dict: + if not values: + return {"mean": None, "p50": None, "p90": None, "p95": None, "p99": None} + return { + "mean": round(sum(values) / len(values), 2), + "p50": round(percentile(values, 50), 2), + "p90": round(percentile(values, 90), 2), + "p95": round(percentile(values, 95), 2), + "p99": round(percentile(values, 99), 2), + } + + aggregates = { + "success": success, + "failed": failed, + "input_tokens": sum(input_tokens), + "output_tokens": sum(output_tokens), + "latencies": { + "e2e_ms": latency_bucket(e2es), + "ttft_ms": latency_bucket(ttfts), + "tpot_ms": latency_bucket(tpots), + "itl_ms": latency_bucket(itls), + }, + } + + return raw_requests, aggregates + + +def parse_scenario(result_root: Path, raw_file: Path) -> dict | None: + """Parse one raw output file into a scenario dict.""" + # Filename: {chip}_{engine}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl + parts = raw_file.stem.split("_") + if len(parts) < 6: + return None + try: + concurrency = int(parts[-3]) + input_len = int(parts[-2]) + output_len = int(parts[-1]) + except ValueError: + return None + + log_file = result_root / "logs" / f"sglang_c{concurrency}_i{input_len}_o{output_len}.log" + summary = {} + if log_file.exists(): + summary = parse_summary_log(log_file.read_text(encoding="utf-8", errors="replace")) + + raw_requests, aggregates = parse_jsonl(raw_file) + + # For newer sglang aggregate JSON, success/failed are not present in the + # raw output file. Override them from the human-readable summary log when + # it is available. + if summary.get("successful_requests") is not None: + aggregates["success"] = int(summary["successful_requests"]) + # The summary log only reports successes; assume failures are zero + # unless the aggregate JSON already provided a non-zero failed count. + if not aggregates.get("failed"): + aggregates["failed"] = 0 + + scenario = { + "name": f"c{concurrency}_i{input_len}_o{output_len}", + "concurrency": concurrency, + "input_len": input_len, + "output_len": output_len, + "success": aggregates["success"], + "failed": aggregates["failed"], + "duration_s": summary.get("benchmark_duration_s"), + "request_throughput": summary.get("request_throughput"), + "input_token_throughput": summary.get("input_token_throughput"), + "output_token_throughput": summary.get("output_token_throughput"), + "total_token_throughput": summary.get("total_token_throughput"), + "accept_length": None, + "latencies": aggregates["latencies"], + "raw_requests": raw_requests[:100] if len(raw_requests) <= 100 else None, + } + + return scenario + + +def main() -> None: + result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") + raw_dir = result_root / "raw_outputs" + json_path = result_root / "results.json" + report_path = result_root / "report.md" + + if not json_path.exists(): + raise SystemExit(f"metadata results.json not found: {json_path}") + + with open(json_path, "r", encoding="utf-8") as f: + data = json.load(f) + + scenarios = [] + if raw_dir.exists(): + for raw_file in sorted(raw_dir.glob("*.jsonl")): + scenario = parse_scenario(result_root, raw_file) + if scenario: + scenarios.append(scenario) + + data["scenarios"] = scenarios + + with open(json_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + # Generate report.md + with open(report_path, "w", encoding="utf-8") as f: + meta = data["metadata"] + f.write(f"# Benchmark Report: {meta['experiment']}\n\n") + f.write("## Metadata\n\n") + f.write(f"- **Run ID**: {meta['run_id']}\n") + f.write(f"- **Timestamp**: {meta['timestamp']}\n") + f.write(f"- **Chip/Accelerator**: {meta.get('accelerator', '')} / {meta.get('chip', '')}\n") + f.write(f"- **Engine/Backend**: {meta.get('engine', '')} / {meta.get('backend', '')}\n") + f.write(f"- **Hardware**: {meta.get('hardware', '')}\n") + f.write(f"- **Model**: {meta['model']}\n") + f.write(f"- **Git Commit**: {meta.get('git_commit', 'unknown')}\n") + f.write("\n## Results\n\n") + f.write( + "| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | " + "TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 |\n" + ) + f.write( + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n" + ) + for s in scenarios: + lat = s["latencies"] + f.write( + f"| {s['name']} " + f"| {s['concurrency']} " + f"| {s['input_len']}/{s['output_len']} " + f"| {s['success']} " + f"| {s['failed']} " + f"| {s.get('request_throughput') or ''} " + f"| {s.get('output_token_throughput') or ''} " + f"| {lat['ttft_ms']['p50'] or ''} " + f"| {lat['ttft_ms']['p99'] or ''} " + f"| {lat['tpot_ms']['p50'] or ''} " + f"| {lat['tpot_ms']['p99'] or ''} " + f"| {lat['e2e_ms']['p99'] or ''} |\n" + ) + + print(f"Updated {json_path}") + print(f"Wrote {report_path}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_p800_256k_4k_probe/run_bench.sh b/experiments/dsv4_p800_256k_4k_probe/run_bench.sh new file mode 100755 index 0000000..c37a0fa --- /dev/null +++ b/experiments/dsv4_p800_256k_4k_probe/run_bench.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/server_docker.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/bench_client_docker.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" +RAW_DIR="${RESULT_ROOT}/raw_outputs" +LOG_DIR="${RESULT_ROOT}/logs" + +ensure_result_root "$RESULT_ROOT" +log_init "${LOG_DIR}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "result_root=${RESULT_ROOT}" +log "platform=${PLATFORM}" +log "chip=${CHIP}" +log "accelerator=${ACCELERATOR}" +log "engine=${ENGINE}" +log "hardware=${HARDWARE}" +log "server_mode=${SERVER_MODE}" +log "extra_launch_args=${SGLANG_EXTRA_LAUNCH_ARGS:-}" + +# Write initial metadata for this run. +METADATA_JSON="${RESULT_ROOT}/results.json" +write_metadata_json \ + "$METADATA_JSON" \ + "$EXPERIMENT_NAME" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "$BACKEND" \ + "$ENGINE" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "" \ + "Kunlun P800 256k-input 4k-output OOM probe for DeepSeek-V4-Flash-INT8" + +# Embed config into metadata. +"${PYTHON:-python3}" - "$METADATA_JSON" <<'PY' >/dev/null +import json +import sys + +path = sys.argv[1] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["config"] = { + "tp": 8, + "xpu_visible_devices": "0,1,2,3,4,5,6,7", + "context_length": 270000, + "server_mode": "w8a8_int8_baseline", + "server_start_script": f"experiments/{data['metadata']['experiment']}/start_server.sh", + "extra_launch_args": "--context-length 270000 --max-running-requests 4", + "scenarios": ["2 262144 4096"], + "num_prompts": 4 +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY + +# Allow the benchmark client to run in a different container than the managed +# server (useful when reusing an already-running deployment). +BENCH_CONTAINER_NAME="${BENCH_CONTAINER_NAME:-${CONTAINER_NAME}}" +if [[ -n "${SKIP_MANAGE_SERVER:-}" && "$BENCH_CONTAINER_NAME" != "$CONTAINER_NAME" ]]; then + log "using benchmark client container: ${BENCH_CONTAINER_NAME}" + CONTAINER_NAME="$BENCH_CONTAINER_NAME" +fi + +# Start server unless SKIP_MANAGE_SERVER is set. +if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then + log "SKIP_MANAGE_SERVER is set, assuming server is already running on port ${PORT}" + if ! health_check 127.0.0.1 "${PORT}" 30; then + log "ERROR: no healthy server found at port ${PORT}" + exit 1 + fi +else + docker_server_start "${MODEL_PATH}" "${PORT}" "${LOG_DIR}/server.outer.log" "${SERVER_MODE}" "${SGLANG_EXTRA_LAUNCH_ARGS:-}" +fi + +on_exit() { + local code=$? + if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then + docker_server_stop + fi + log "orchestrator exiting with code=${code}" + exit "$code" +} +trap on_exit EXIT + +log "===== BENCHMARK START =====" + +# Optional small warmup to exercise the long-context path. Disabled by default +# because the probe scenario itself is already small (4 prompts). +if [[ -z "${SKIP_WARMUP:-}" ]]; then + log "running small warmup (1 prompt, 64k input, 256 output)" + tmp_warmup_file="/tmp/bench_outputs/${CHIP}_${ENGINE}_$(date '+%m%d')_warmup_1_65536_256.jsonl" + host_warmup_file="${RAW_DIR}/${CHIP}_${ENGINE}_$(date '+%m%d')_warmup_1_65536_256.jsonl" + detail_log="${LOG_DIR}/warmup.log" + + if WARMUP=0 NUM_PROMPTS=1 run_random_case \ + "$BACKEND" "$PORT" "$MODEL_PATH" "$tmp_warmup_file" \ + 1 65536 256 1 "${DATASET_PATH:-}" \ + > "$detail_log" 2>&1; then + docker cp "${CONTAINER_NAME}:${tmp_warmup_file}" "$host_warmup_file" 2>/dev/null || true + log "warmup completed" + else + log "WARNING: warmup failed; continuing with main scenario" + fi +fi + +for scenario in "${SCENARIOS[@]}"; do + # Allow SCENARIOS to be passed as a string like "(2 262144 4096)" from env. + scenario="${scenario//[()]/}" + read -r concurrency input_len output_len <<< "$scenario" + + tmp_output_file="/tmp/bench_outputs/${CHIP}_${ENGINE}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + host_output_file="${RAW_DIR}/${CHIP}_${ENGINE}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + detail_log="${LOG_DIR}/${BACKEND}_c${concurrency}_i${input_len}_o${output_len}.log" + + log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${NUM_PROMPTS}" + + if run_random_case \ + "$BACKEND" "$PORT" "$MODEL_PATH" "$tmp_output_file" \ + "$concurrency" "$input_len" "$output_len" "$NUM_PROMPTS" \ + "${DATASET_PATH:-}" \ + > "$detail_log" 2>&1; then + docker cp "${CONTAINER_NAME}:${tmp_output_file}" "$host_output_file" || { + log "ERROR: failed to copy ${tmp_output_file} from container" + continue + } + log "finished scenario: output=${host_output_file}" + else + log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" + fi +done + +log "===== BENCHMARK DONE =====" +log "parsing results..." +"${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" || log "WARNING: parse_results.py failed" + +log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_p800_256k_4k_probe/start_server.sh b/experiments/dsv4_p800_256k_4k_probe/start_server.sh new file mode 100755 index 0000000..c0894ed --- /dev/null +++ b/experiments/dsv4_p800_256k_4k_probe/start_server.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/server_docker.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +log_init "${RESULT_ROOT:-/tmp/${EXPERIMENT_NAME}}/logs/start_server.log" +log "starting server for ${EXPERIMENT_NAME}" +log "model: ${MODEL_PATH}" +log "port: ${PORT}" +log "server mode: ${SERVER_MODE}" +log "extra launch args: ${SGLANG_EXTRA_LAUNCH_ARGS:-}" + +docker_server_start "${MODEL_PATH}" "${PORT}" "${LOG_DIR}/server.outer.log" "${SERVER_MODE}" "${SGLANG_EXTRA_LAUNCH_ARGS:-}" diff --git a/experiments/dsv4_p800_sglang/README.md b/experiments/dsv4_p800_sglang/README.md index e88ec87..4c8b222 100644 --- a/experiments/dsv4_p800_sglang/README.md +++ b/experiments/dsv4_p800_sglang/README.md @@ -24,6 +24,10 @@ SCENARIOS=("32 512 256" "128 512 256" "256 512 256" "512 512 256") \ NUM_PROMPTS=512 \ PLATFORM=kunlun_p800 \ bash experiments/dsv4_p800_sglang/run_bench.sh + +# Use the tuned config with EAGLE speculative decoding instead of the default baseline +SERVER_MODE=w8a8_int8 PLATFORM=kunlun_p800 \ +bash experiments/dsv4_p800_sglang/run_bench.sh ``` ## Files diff --git a/experiments/dsv4_p800_sglang/config.env b/experiments/dsv4_p800_sglang/config.env index f984eb8..af9a15b 100644 --- a/experiments/dsv4_p800_sglang/config.env +++ b/experiments/dsv4_p800_sglang/config.env @@ -20,9 +20,10 @@ WARMUP="${WARMUP:-0}" NUM_PROMPTS="${NUM_PROMPTS:-10}" # Server launch mode for P800 SGLang. -# - "fp8" : non-INT8 DeepSeek-V4-Flash checkpoint -# - "w8a8_int8": DeepSeek-V4-Flash-INT8 checkpoint -SERVER_MODE="${SERVER_MODE:-w8a8_int8}" +# - "fp8" : non-INT8 DeepSeek-V4-Flash checkpoint +# - "w8a8_int8" : DeepSeek-V4-Flash-INT8 checkpoint with EAGLE speculative decoding +# - "w8a8_int8_baseline": DeepSeek-V4-Flash-INT8 checkpoint without speculative decoding +SERVER_MODE="${SERVER_MODE:-w8a8_int8_baseline}" # Scenario list: "concurrency input_len output_len" (one per line). # Default to a single smoke-test scenario for fast validation. Override to run diff --git a/scripts/common/server_docker.sh b/scripts/common/server_docker.sh index 02e6fec..521b85a 100755 --- a/scripts/common/server_docker.sh +++ b/scripts/common/server_docker.sh @@ -54,6 +54,7 @@ docker_server_start() { local port="${2:-$DEFAULT_PORT}" local server_log="${3:-${RESULT_ROOT:-/tmp}/logs/server.outer.log}" local mode="${4:-fp8}" + local extra_args="${5:-${SGLANG_EXTRA_LAUNCH_ARGS:-}}" local container="${CONTAINER_NAME}" local image="${DOCKER_IMAGE}" @@ -145,10 +146,20 @@ docker_server_start() { local launch_args if [[ "$mode" == "w8a8_int8" ]]; then launch_args="--host 0.0.0.0 --port ${port} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 16384 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging" + elif [[ "$mode" == "w8a8_int8_baseline" ]]; then + # Baseline w8a8_int8 launch without speculative decoding and without + # hard-coded chunked-prefill / memory-fraction settings. Matches the + # long-context probe tuning used for P800. + launch_args="--host 0.0.0.0 --port ${port} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging" else launch_args="--host 0.0.0.0 --port ${port} --model-path /models --quantization fp8 --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.85 --max-prefill-tokens 16384 --context-length 65536 --max-running-requests 8 --max-total-tokens 524288 --tensor-parallel-size 8 --ep-size 8 --moe-runner-backend deep_gemm --disable-shared-experts-fusion --kv-cache-dtype float16 --disable-piecewise-cuda-graph --disable-cuda-graph --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --enable-metrics --enable-request-time-stats-logging" fi + if [[ -n "$extra_args" ]]; then + launch_args="${launch_args} ${extra_args}" + log "extra launch args: ${extra_args}" + fi + # Build the full server bootstrap command and base64-encode it to avoid # host-shell quoting hell. local server_cmd