diff --git a/README.md b/README.md index 2ce958d..b135e7a 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ | DSV4 H200 SGLang baseline | `experiments/dsv4_h200_sglang/run_bench.sh` | NVIDIA H200 + native SGLang + DeepSeek-V4-Flash | | DSV4 H200 SGLang vs vLLM | `experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh` | NVIDIA H200 上 SGLang 与 vLLM 控制变量对比(TP=8,最长 200k 上下文) | | DSV4 H200 vLLM DSpark vs default | `experiments/dsv4_h200_vllm_dspark_vs_default/run_bench.sh` | vLLM 开启 DSpark 投机解码 vs 默认配置,验证 TTFT 差异 | +| DSV4 H200 max context length | `experiments/dsv4_h200_max_context_length/run_bench.sh` | 探索 SGLang / vLLM 在 H200 上能支持的最大输入长度 | ### 旧结构(scripts/ + bench_results/) @@ -76,6 +77,12 @@ bash experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh bash experiments/dsv4_h200_vllm_dspark_vs_default/run_bench.sh ``` +### H200 max context length + +```bash +bash experiments/dsv4_h200_max_context_length/run_bench.sh +``` + ### DSpark grid(旧结构) ```bash diff --git a/experiments/dsv4_h200_max_context_length/README.md b/experiments/dsv4_h200_max_context_length/README.md new file mode 100644 index 0000000..583d098 --- /dev/null +++ b/experiments/dsv4_h200_max_context_length/README.md @@ -0,0 +1,59 @@ +# DSV4 H200 最大上下文长度探索实验 + +在 8x NVIDIA H200 上,逐步增大输入长度,测试 **SGLang** 与 **vLLM** serving DeepSeek-V4-Flash 时各自能稳定处理的最长上下文。 + +## 目的 + +- 找到 SGLang / vLLM 在 H200 TP=8 配置下,单请求最多能支持多长的输入 token。 +- 每个长度只跑 1 个请求、输出 1 个 token,尽量减少 decode 阶段显存干扰。 +- 每次测试单独启动 server,并把 `max-model-len` / `context-length` 刚好设为 `target_len + 1024`,避免预分配过大。 + +## 默认候选长度 + +```text +65536, 131072, 262144, 393216, 524288, 655360, 786432, 917504, 1048576 +``` + +可以通过环境变量覆盖: + +```bash +CANDIDATE_LENS=(65536 131072 262144 524288) bash run_bench.sh +``` + +## 控制变量 + +| 维度 | SGLang | vLLM | +|---|---|---| +| 硬件 | 8x H200,TP=8 | 同上 | +| 输入长度 | `--random-input-len $target_len --random-range-ratio 1.0` | 同上 | +| 输出长度 | 1 | 1 | +| 并发 / 请求数 | 1 / 1 | 1 / 1 | +| 压测客户端 | `sglang.bench_serving --backend vllm` | 同上 | + +> `--random-range-ratio 1.0` 确保输入长度固定为 `target_len`,而不是默认的 `[1, target_len]` 均匀分布。 + +## 快速运行 + +```bash +bash experiments/dsv4_h200_max_context_length/run_bench.sh +``` + +结果保存在 `experiments/dsv4_h200_max_context_length/results//`: + +``` +results// +├── results.json +├── report.md +└── raw_outputs/ # gitignored +``` + +## 产物说明 + +- `results.json`:每个候选长度的尝试记录,包括 `success`、TTFT/TPOT/E2E 指标、`server_args`。 +- `report.md`:汇总表格,以及每个 backend 最终支持的最大输入长度。 + +## 环境 + +- SGLM server: `/data/user1/yy/envs/sglang` +- vLLM server: `/data/user1/yy/envs/vllm` +- Benchmark client: `/data/user1/yy/envs/sglang` diff --git a/experiments/dsv4_h200_max_context_length/config.env b/experiments/dsv4_h200_max_context_length/config.env new file mode 100644 index 0000000..180129d --- /dev/null +++ b/experiments/dsv4_h200_max_context_length/config.env @@ -0,0 +1,48 @@ +# Common configuration for the max context length exploration on H200. +# All values can be overridden via environment variables. + +EXPERIMENT="dsv4_h200_max_context_length" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +# Ports must differ from other running experiments. +SGLANG_PORT="${SGLANG_PORT:-30011}" +VLLM_PORT="${VLLM_PORT:-30012}" + +# Virtual environments. +VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}" +VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}" +VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" + +# Hardware: use all 8 H200 cards for both backends. +export CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" +TP=8 + +# We want exact input lengths, so fix the random range ratio to 1.0. +OUTPUT_LEN="${OUTPUT_LEN:-1}" +NUM_PROMPTS="${NUM_PROMPTS:-1}" +MAX_CONCURRENCY="${MAX_CONCURRENCY:-1}" +REQUEST_RATE="${REQUEST_RATE:-10000}" + +# Padding added to target length when setting server max context/model length. +CONTEXT_PAD="${CONTEXT_PAD:-1024}" + +# Candidate input lengths (tokens). Override via CANDIDATE_LENS env var. +if [[ -z "${CANDIDATE_LENS:-}" ]]; then + declare -a CANDIDATE_LENS=( + 65536 + 131072 + 262144 + 393216 + 524288 + 655360 + 786432 + 917504 + 1048576 + ) +fi + +# Server start scripts bundled with this experiment. +SGLANG_START_SCRIPT="${SCRIPT_DIR:-.}/start_sglang.sh" +VLLM_START_SCRIPT="${SCRIPT_DIR:-.}/start_vllm.sh" diff --git a/experiments/dsv4_h200_max_context_length/extract_metrics.py b/experiments/dsv4_h200_max_context_length/extract_metrics.py new file mode 100755 index 0000000..c2579ce --- /dev/null +++ b/experiments/dsv4_h200_max_context_length/extract_metrics.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Extract a compact metrics dict from a sglang.bench_serving JSONL output. + +Usage: + python3 extract_metrics.py +""" +import json +import sys +from pathlib import Path + + +def parse_jsonl(path: Path) -> dict | None: + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + return json.loads(line) + except json.JSONDecodeError: + continue + return None + + +def compute(data: dict) -> dict: + return { + "success_count": data.get("completed", 0), + "total_count": len(data.get("input_lens", [])), + "duration_s": data.get("duration", 0.0), + "request_throughput": data.get("request_throughput", 0.0), + "input_token_throughput": data.get("input_throughput", 0.0), + "output_token_throughput": data.get("output_throughput", 0.0), + "total_input_tokens": data.get("total_input_tokens", 0), + "total_output_tokens": data.get("total_output_tokens", 0), + "e2e_ms": { + "mean": data.get("mean_e2e_latency_ms", 0.0), + "p50": data.get("median_e2e_latency_ms", 0.0), + "p90": data.get("p90_e2e_latency_ms", 0.0), + "p95": data.get("p95_e2e_latency_ms", 0.0), + "p99": data.get("p99_e2e_latency_ms", 0.0), + }, + "ttft_ms": { + "mean": data.get("mean_ttft_ms", 0.0), + "p50": data.get("median_ttft_ms", 0.0), + "p90": data.get("p90_ttft_ms", 0.0), + "p95": data.get("p95_ttft_ms", 0.0), + "p99": data.get("p99_ttft_ms", 0.0), + }, + "tpot_ms": { + "mean": data.get("mean_tpot_ms", 0.0), + "p50": data.get("median_tpot_ms", 0.0), + "p90": data.get("p90_tpot_ms", 0.0), + "p95": data.get("p95_tpot_ms", 0.0), + "p99": data.get("p99_tpot_ms", 0.0), + }, + "itl_ms": { + "mean": data.get("mean_itl_ms", 0.0), + "p50": data.get("median_itl_ms", 0.0), + "p90": data.get("p90_itl_ms", 0.0), + "p95": data.get("p95_itl_ms", 0.0), + "p99": data.get("p99_itl_ms", 0.0), + }, + } + + +def main(): + path = Path(sys.argv[1]) + data = parse_jsonl(path) + if data is None: + raise SystemExit("No valid JSON found") + print(json.dumps(compute(data), indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_max_context_length/parse_results.py b/experiments/dsv4_h200_max_context_length/parse_results.py new file mode 100755 index 0000000..0c497d3 --- /dev/null +++ b/experiments/dsv4_h200_max_context_length/parse_results.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Generate report.md from results.json for the max context length experiment. + +Usage: + python3 parse_results.py +""" +import json +import sys +from pathlib import Path + + +def main(): + result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") + results_json = result_root / "results.json" + report_path = result_root / "report.md" + + with open(results_json, "r", encoding="utf-8") as f: + data = json.load(f) + + meta = data["metadata"] + cfg = data["config"] + + with open(report_path, "w", encoding="utf-8") as f: + f.write("# H200 Max Context Length Exploration Report\n\n") + f.write(f"- Result root: `{result_root}`\n") + f.write(f"- Model: `{meta['model']}`\n") + f.write(f"- Hardware: {meta['hardware']}\n") + f.write(f"- TP: {cfg['tp']}\n") + f.write(f"- Output len: {cfg['output_len']}, num prompts: {cfg['num_prompts']}, concurrency: {cfg['max_concurrency']}\n\n") + + f.write("## Per-attempt results\n\n") + f.write("| Backend | Target len | Max model/context len | Status | TTFT mean(ms) | TTFT P95(ms) | E2E mean(ms) | E2E P99(ms) | Error |\n") + f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + max_by_backend = {} + for s in data.get("scenarios", []): + backend = s["backend"] + target = s["target_len"] + status = "✅ pass" if s["success"] else "❌ fail" + metrics = s.get("metrics") or {} + ttft = metrics.get("ttft_ms", {}) + e2e = metrics.get("e2e_ms", {}) + f.write( + f"| {backend} | {target} | {s['max_model_len']} | {status} | " + f"{ttft.get('mean', 0):.2f} | {ttft.get('p95', 0):.2f} | " + f"{e2e.get('mean', 0):.2f} | {e2e.get('p99', 0):.2f} | " + f"{s.get('error', '') or ''} |\n" + ) + if s["success"] and (backend not in max_by_backend or target > max_by_backend[backend]): + max_by_backend[backend] = target + + f.write("\n## Maximum supported input length\n\n") + for backend in ("sglang", "vllm"): + max_len = max_by_backend.get(backend, 0) + f.write(f"- **{backend}**: {max_len} tokens\n") + f.write("\n") + + print(f"Wrote report to {report_path}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_max_context_length/run_bench.sh b/experiments/dsv4_h200_max_context_length/run_bench.sh new file mode 100755 index 0000000..bc1515d --- /dev/null +++ b/experiments/dsv4_h200_max_context_length/run_bench.sh @@ -0,0 +1,269 @@ +#!/usr/bin/env bash +# Max context length exploration for SGLang and vLLM on H200. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" +LOG_DIR="${RESULT_ROOT}/logs" +RAW_DIR="${RESULT_ROOT}/raw_outputs" + +ensure_result_root "$RESULT_ROOT" +log_init "${LOG_DIR}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "platform=${PLATFORM}" +log "hardware=${HARDWARE}" +log "model=${MODEL_PATH}" +log "candidate_lengths=${CANDIDATE_LENS[*]}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +is_server_healthy() { + local port="$1" + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 +} + +stop_server() { + local backend="$1" + local pid_file="/data/user1/yy/${EXPERIMENT_NAME}_${backend}.pid" + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping ${backend} server pid=${pid}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + sleep 2 +} + +start_server() { + local backend="$1" + local target_len="$2" + local start_script port + if [[ "$backend" == "sglang" ]]; then + start_script="${SGLANG_START_SCRIPT}" + port="$SGLANG_PORT" + else + start_script="${VLLM_START_SCRIPT}" + port="$VLLM_PORT" + fi + + log "starting ${backend} server (target_len=${target_len}) with ${start_script}" + bash "${start_script}" "$target_len" >> "${LOG_DIR}/${backend}_${target_len}.server.outer.log" 2>&1 + + if ! is_server_healthy "$port"; then + log "error: ${backend} server (target_len=${target_len}) failed to become healthy" + return 1 + fi + log "${backend} server is healthy on port ${port}" +} + +build_server_args() { + local backend="$1" + local target_len="$2" + local max_len=$(( target_len + CONTEXT_PAD )) + if [[ "$backend" == "sglang" ]]; then + echo "sglang serve --trust-remote-code --model-path $MODEL_PATH --tp $TP --moe-runner-backend marlin --context-length $max_len --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port $SGLANG_PORT" + else + echo "vllm serve $MODEL_PATH --trust-remote-code --tensor-parallel-size $TP --kv-cache-dtype fp8 --max-model-len $max_len --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port $VLLM_PORT" + fi +} + +append_scenario() { + local scenario_json="$1" + "${VENV_CLIENT}/bin/python" - "$RESULT_JSON" "$scenario_json" <<'PY' +import json +import sys + +results_path, scenario_json = sys.argv[1], sys.argv[2] +with open(results_path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["scenarios"].append(json.loads(scenario_json)) +with open(results_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY +} + +run_backend() { + local backend="$1" + local port + if [[ "$backend" == "sglang" ]]; then + port="$SGLANG_PORT" + else + port="$VLLM_PORT" + fi + + log "===== ${backend} max context exploration START =====" + + for target_len in "${CANDIDATE_LENS[@]}"; do + local max_len=$(( target_len + CONTEXT_PAD )) + local output_file="${RAW_DIR}/${backend}_ctx_${target_len}.jsonl" + local detail_log="${LOG_DIR}/${backend}_ctx_${target_len}.log" + + log "trying ${backend} target_len=${target_len} (max_model/context_len=${max_len})" + + stop_server "$backend" + if ! start_server "$backend" "$target_len"; then + local err="server failed to start" + log "ERROR: ${err}" + append_scenario "$(jq -n \ + --arg backend "$backend" \ + --argjson target_len "$target_len" \ + --argjson max_model_len "$max_len" \ + --arg server_args "$(build_server_args "$backend" "$target_len")" \ + --arg error "$err" \ + '{backend:$backend, target_len:$target_len, max_model_len:$max_model_len, success:false, server_args:$server_args, metrics:null, error:$error}')" + break + fi + + local bench_rc=0 + "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \ + --backend vllm \ + --host 127.0.0.1 \ + --port "$port" \ + --dataset-name random \ + --random-input-len "$target_len" \ + --random-output-len "$OUTPUT_LEN" \ + --random-range-ratio 1.0 \ + --num-prompts "$NUM_PROMPTS" \ + --max-concurrency "$MAX_CONCURRENCY" \ + --request-rate "$REQUEST_RATE" \ + --output-file "$output_file" \ + --output-details \ + > "$detail_log" 2>&1 || bench_rc=$? + + if [[ "$bench_rc" -ne 0 ]]; then + local err="bench_serving exited with code ${bench_rc}; see ${detail_log}" + log "ERROR: ${backend} target_len=${target_len} failed; ${err}" + append_scenario "$(jq -n \ + --arg backend "$backend" \ + --argjson target_len "$target_len" \ + --argjson max_model_len "$max_len" \ + --arg server_args "$(build_server_args "$backend" "$target_len")" \ + --arg error "$err" \ + '{backend:$backend, target_len:$target_len, max_model_len:$max_model_len, success:false, server_args:$server_args, metrics:null, error:$error}')" + stop_server "$backend" + break + fi + + # Verify the request actually completed. + local completed + completed="$(${VENV_CLIENT}/bin/python -c " +import json +with open('${output_file}') as f: + for line in f: + data = json.loads(line) + print(data.get('completed', 0)) + break +")" + if [[ "${completed:-0}" -lt "$NUM_PROMPTS" ]]; then + local err="only ${completed}/${NUM_PROMPTS} requests completed" + log "ERROR: ${backend} target_len=${target_len} ${err}" + append_scenario "$(jq -n \ + --arg backend "$backend" \ + --argjson target_len "$target_len" \ + --argjson max_model_len "$max_len" \ + --arg server_args "$(build_server_args "$backend" "$target_len")" \ + --arg error "$err" \ + '{backend:$backend, target_len:$target_len, max_model_len:$max_model_len, success:false, server_args:$server_args, metrics:null, error:$error}')" + stop_server "$backend" + break + fi + + local metrics_json + metrics_json="$(${VENV_CLIENT}/bin/python "${SCRIPT_DIR}/extract_metrics.py" "$output_file")" + + append_scenario "$(jq -n \ + --arg backend "$backend" \ + --argjson target_len "$target_len" \ + --argjson max_model_len "$max_len" \ + --arg server_args "$(build_server_args "$backend" "$target_len")" \ + --argjson metrics "$metrics_json" \ + '{backend:$backend, target_len:$target_len, max_model_len:$max_model_len, success:true, server_args:$server_args, metrics:$metrics, error:null}')" + + log "${backend} target_len=${target_len} succeeded" + stop_server "$backend" + done + + log "===== ${backend} max context exploration DONE =====" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +mkdir -p "$RAW_DIR" "$LOG_DIR" + +RESULT_JSON="${RESULT_ROOT}/results.json" +write_metadata_json \ + "$RESULT_JSON" \ + "$EXPERIMENT_NAME" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "sglang+vllm" \ + "sglang+vllm" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$VENV_CLIENT" \ + "H200 max context length exploration for DeepSeek-V4-Flash" + +# Embed config. +"${VENV_CLIENT}/bin/python" - "$RESULT_JSON" <<'PY' +import json +import sys + +path = sys.argv[1] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["config"] = { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "output_len": 1, + "num_prompts": 1, + "max_concurrency": 1, + "request_rate": 10000, + "context_pad": 1024, +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY + +# Ensure jq is available. +if ! command -v jq >/dev/null 2>&1; then + log "ERROR: jq is required for JSON manipulation in this script" + exit 1 +fi + +stop_server sglang +stop_server vllm + +run_backend sglang +run_backend vllm + +log "parsing results" +"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || { + log "WARNING: parse_results.py failed; see ${LOG_DIR}/parse.log" +} + +log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_h200_max_context_length/start_sglang.sh b/experiments/dsv4_h200_max_context_length/start_sglang.sh new file mode 100755 index 0000000..a82d5db --- /dev/null +++ b/experiments/dsv4_h200_max_context_length/start_sglang.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Start SGLang server for a specific target context length. +set -e + +TARGET_LEN="${1:-}" +if [[ -z "$TARGET_LEN" ]]; then + echo "Usage: $0 " + exit 1 +fi + +cd /data/user1/yy +mkdir -p logs + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +export PATH="${VENV_SGLANG}/bin:$PATH" +export PYTHONUNBUFFERED=1 +export SGLANG_LOG_LEVEL=info +export TMPDIR=/data/user1/yy/tmp +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +MAX_CONTEXT_LEN=$(( TARGET_LEN + CONTEXT_PAD )) + +LOG="/data/user1/yy/logs/dsv4_h200_max_context_length_sglang_${TARGET_LEN}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="/data/user1/yy/dsv4_h200_max_context_length_sglang.pid" + +rm -f "$PID_FILE" + +echo "=== Starting SGLang server (TP=$TP, target_input_len=$TARGET_LEN, max_context_len=$MAX_CONTEXT_LEN) ===" +echo "Model: $MODEL_PATH" +echo "Port: $SGLANG_PORT" +echo "Log: $LOG" + +nohup sglang serve \ + --trust-remote-code \ + --model-path "$MODEL_PATH" \ + --tp "$TP" \ + --moe-runner-backend marlin \ + --context-length "$MAX_CONTEXT_LEN" \ + --max-running-requests 1 \ + --mem-fraction-static 0.88 \ + --host 0.0.0.0 \ + --port "$SGLANG_PORT" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${SGLANG_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: SGLang server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: SGLang server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/dsv4_h200_max_context_length/start_vllm.sh b/experiments/dsv4_h200_max_context_length/start_vllm.sh new file mode 100755 index 0000000..dce6692 --- /dev/null +++ b/experiments/dsv4_h200_max_context_length/start_vllm.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Start vLLM server for a specific target context length. +set -e + +TARGET_LEN="${1:-}" +if [[ -z "$TARGET_LEN" ]]; then + echo "Usage: $0 " + exit 1 +fi + +cd /data/user1/yy +mkdir -p logs + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +VENV="${VENV_VLLM}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR=/data/user1/yy/tmp +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +MAX_MODEL_LEN=$(( TARGET_LEN + CONTEXT_PAD )) + +LOG="/data/user1/yy/logs/dsv4_h200_max_context_length_vllm_${TARGET_LEN}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="/data/user1/yy/dsv4_h200_max_context_length_vllm.pid" + +rm -f "$PID_FILE" + +echo "=== Starting vLLM server (TP=$TP, target_input_len=$TARGET_LEN, max_model_len=$MAX_MODEL_LEN) ===" +echo "Model: $MODEL_PATH" +echo "Port: $VLLM_PORT" +echo "Log: $LOG" + +nohup vllm serve "$MODEL_PATH" \ + --trust-remote-code \ + --tensor-parallel-size "$TP" \ + --kv-cache-dtype fp8 \ + --max-model-len "$MAX_MODEL_LEN" \ + --max-num-seqs 1 \ + --block-size 256 \ + --gpu-memory-utilization 0.90 \ + --tokenizer-mode deepseek_v4 \ + --reasoning-parser deepseek_v4 \ + --no-disable-hybrid-kv-cache-manager \ + --disable-uvicorn-access-log \ + --port "$VLLM_PORT" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: vLLM server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/results.json b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/results.json new file mode 100644 index 0000000..92be18f --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/sglang/results.json @@ -0,0 +1,29 @@ +{ + "metadata": { + "experiment": "dsv4_h200_sglang_vs_vllm_sglang", + "run_id": "20260708-083346", + "timestamp": "2026-07-08T08:33:51+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "sglang", + "engine": "sglang", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh", + "env": "/data/user1/yy/envs/sglang", + "git_commit": "e9ed45c", + "git_dirty": "dirty", + "description": "H200 sglang TP=8 comparison benchmark for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "phase1_max_model_len": 32768, + "phase2_max_model_len": 210000, + "backend": "sglang", + "server_start_script": "experiments/dsv4_h200_sglang_vs_vllm/start_sglang.sh", + "phase1_server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 32768 --max-running-requests 256 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30006", + "phase2_server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 210000 --max-running-requests 2 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30006" + }, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/results.json b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/results.json new file mode 100644 index 0000000..d1bc61d --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-083346/vllm/results.json @@ -0,0 +1,29 @@ +{ + "metadata": { + "experiment": "dsv4_h200_sglang_vs_vllm_vllm", + "run_id": "20260708-083346", + "timestamp": "2026-07-08T08:33:51+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "e9ed45c", + "git_dirty": "dirty", + "description": "H200 vllm TP=8 comparison benchmark for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "phase1_max_model_len": 32768, + "phase2_max_model_len": 210000, + "backend": "vllm", + "server_start_script": "experiments/dsv4_h200_sglang_vs_vllm/start_vllm.sh", + "phase1_server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 32768 --max-num-seqs 256 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30005", + "phase2_server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 210000 --max-num-seqs 2 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30005" + }, + "scenarios": [] +} \ No newline at end of file