diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env new file mode 100644 index 0000000..4085397 --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env @@ -0,0 +1,49 @@ +# Adaptive concurrency search settings. +# +# For each fixed (TP, DP, ISL, DSL), probe: +# C = start, start * multiplier, ... up to max +# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for +# PLATEAU_PATIENCE consecutive points. + +SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" +SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}" +NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" + +# A gain below 2% is treated as throughput saturation. Two consecutive +# low-gain points prevent one noisy measurement from stopping the search. +TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" +PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" + +# Keep the same random workload semantics as the fixed matrix baseline. +# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times +# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to +# random-ids to use generated token IDs without a ShareGPT seed dataset. +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed +# ISL/DSL points; lower values intentionally benchmark a length distribution. +RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" +# Before each measured point, warm up with the same concurrency so lazy kernel +# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no +# cap; set a positive cap only when very high-concurrency warmup is impractical. +BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}" + +# Reject a point if the completed request count or actual token lengths do not +# match the requested workload. +INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}" +OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}" + +MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}" +SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Optional space-separated filters, useful for smoke tests: +# TP_LIST="8" ISL_LIST="1024" DSL_LIST="128" +TP_LIST="${TP_LIST:-}" +ISL_LIST="${ISL_LIST:-}" +DSL_LIST="${DSL_LIST:-}" + +DRY_RUN="${DRY_RUN:-0}" +# Counts ISL/DSL shapes per TP/DP config, not individual concurrency probes. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/config.env b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/config.env new file mode 100644 index 0000000..c963f5c --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/config.env @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# TP×DP matrix experiment for DeepSeek-V4-Flash on RTX 6000D (8 GPUs) using SGLang. +# Tests SGLang with three parallel configurations: +# TP=2, DP=4 -> 2 GPUs per replica, 4 replicas +# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas +# TP=8, DP=1 -> 8 GPUs, no data parallelism + +EXPERIMENT="dsv4_pro6000_sglang_tp_dp_matrix" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/hf_models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +SGLANG_PORT="${SGLANG_PORT:-30031}" + +# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.) +# and the benchmark client. Defaults to the system python3 if the sglang venv +# does not exist on the host. +VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}" + +# Run the benchmark client natively (0) or inside Docker (1). +USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" + +# Runtime working directory for logs, pid files, and tmp. Defaults to a local +# directory under this experiment so the benchmark is self-contained. +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" + +# Parallel configurations to test. Format: "TP DP" +declare -a PARALLEL_CONFIGS=( + "2 4" + "4 2" + "8 1" +) + +# SGLang server settings +CONTEXT_LENGTH="${CONTEXT_LENGTH:-1048576}" +MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-128}" +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.88}" +MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}" + +# Deployment switch. 0 = native sglang venv, 1 = Docker. +USE_DOCKER="${USE_DOCKER:-1}" +DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" + +# Dataset used by sglang.bench_serving --dataset-name random. +# The random sampler needs a ShareGPT-style JSON file locally; it falls back to +# downloading from HuggingFace, which usually fails on offline nodes. +DATASET_PATH="${DATASET_PATH:-/data/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}" + +# Matrix and concurrency rules are defined in matrix.json by default. +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" + +# Sampling density for concurrency. +# 0 = use the default heuristic in generate_scenarios.py (6-8 points). +# 2 = only test the low and high endpoints. +export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}" + +# Per-scenario timeout to avoid hangs (seconds). +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" + +# GPU memory sampling interval (seconds). +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Dry-run mode: if 1, only log the server args and scenario plan without starting +# any server or sending requests. +DRY_RUN="${DRY_RUN:-0}" + +# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh new file mode 100755 index 0000000..90f7827 --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/DSL shape. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="sglang" +ENGINE_PORT="$SGLANG_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then + log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}" + docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true + sleep 5 + docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true + fi + elif kill -0 "$pid" 2>/dev/null; then + log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + if [[ -z "${CONTAINER_NAME:-}" ]]; then + docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + fi + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + sglang serve --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$tp" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--dp-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + log "starting sglang server tp=${tp} dp=${dp}" + if [[ -n "${CONTAINER_NAME:-}" ]]; then + bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + else + bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + fi + if ! engine_is_healthy; then + log "ERROR: sglang health check failed tp=${tp} dp=${dp}" + return 1 + fi + if [[ -z "${CONTAINER_NAME:-}" ]]; then + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + fi + log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then + return 0 + fi + if [[ -n "${CONTAINER_NAME:-}" ]]; then + docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null + return $? + fi + return 1 +} + +engine_run_bench() { + local isl="$1" + local dsl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend sglang + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$dsl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + "$DOCKER_IMAGE" \ + python -m sglang.bench_serving "${bench_args[@]}" + else + "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +adaptive_main "$@" diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh new file mode 100755 index 0000000..69a08b0 --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Run SGLang server inside the already-running benchmark container. +# Usage: run_sglang_in_container.sh +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" + +PORT="${SGLANG_PORT:-30031}" +NAME="${EXPERIMENT}_sglang_container" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_in_container_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +# Build server args. +SERVER_ARGS=( + serve + --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$TP" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --dp-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting SGLang server inside container (TP=${TP}, DP=${DP}) ===" +echo "Container: $NAME" +echo "Command: sglang ${SERVER_ARGS_STR}" +echo "Log: $LOG" + +# Kill any existing sglang process inside container first. +docker exec "$NAME" pkill -9 -f "sglang serve" 2>/dev/null || true +sleep 3 + +# Start sglang serve inside the container in background. +# We use nohup so it survives after docker exec returns. +docker exec -d \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + "$NAME" \ + bash -c "nohup sglang ${SERVER_ARGS_STR} > /tmp/sglang_server.log 2>&1 &" + +# Wait for the server process to appear inside container. +sleep 2 +SERVER_PID=$(docker exec "$NAME" pgrep -f "sglang serve" | head -n 1 || true) +if [[ -z "$SERVER_PID" ]]; then + echo "ERROR: sglang server process not found inside container" + docker exec "$NAME" cat /tmp/sglang_server.log 2>/dev/null | tail -50 || true + exit 1 +fi + +echo "Server PID inside container: $SERVER_PID" +echo "$SERVER_PID" > "$PID_FILE" + +echo "Waiting for health on port ${PORT}..." +for i in $(seq 1 360); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${PORT}" + echo "Log: $LOG" + exit 0 + fi + # Check if server process is still alive. + if ! docker exec "$NAME" kill -0 "$SERVER_PID" 2>/dev/null; then + echo "ERROR: SGLang server exited early" + docker exec "$NAME" cat /tmp/sglang_server.log 2>/dev/null | tail -200 || true + exit 1 + fi + echo "Waiting... ($i/360)" + sleep 5 +done + +echo "ERROR: SGLang server not healthy after 360 retries (30 mins)" +docker exec "$NAME" cat /tmp/sglang_server.log 2>/dev/null | tail -200 || true +exit 1 diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_container.sh b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_container.sh new file mode 100755 index 0000000..2cf469c --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_container.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Start a long-running Docker container for SGLang benchmarking. +# The container stays alive (sleep infinity) so we can docker exec into it +# to run different TP×DP configurations without losing DeepGEMM JIT cache. +# +# Usage: start_sglang_container.sh +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" +PORT="${SGLANG_PORT:-30031}" +NAME="${EXPERIMENT}_sglang_container" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_container.pid" + +# Persistent cache directory on host for DeepGEMM JIT kernels. +CACHE_DIR="${RUNTIME_BASE}/cache" +mkdir -p "${CACHE_DIR}/deep_gemm" "${CACHE_DIR}/tvm-ffi" + +# Clean up any stale container. +docker rm -f "$NAME" >/dev/null 2>&1 || true + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_container_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +echo "=== Starting SGLang benchmark container ===" +echo "Image: $IMAGE" +echo "Container name: $NAME" +echo "Host port: $PORT" +echo "Cache dir: $CACHE_DIR" +echo "Log: $LOG" + +# Start a long-running container. +# We override the entrypoint to sleep infinity so the container stays alive. +docker run -d \ + --name "$NAME" \ + --gpus all \ + --ipc host \ + --shm-size 16g \ + --entrypoint /bin/bash \ + -p "${PORT}:${PORT}" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${CACHE_DIR}/deep_gemm:/root/.cache/deep_gemm" \ + -v "${CACHE_DIR}/tvm-ffi:/root/.cache/tvm-ffi" \ + -v "${RUNTIME_BASE}/tmp:/tmp" \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + "$IMAGE" \ + -c "sleep infinity" \ + > "$LOG" 2>&1 + +# Wait a moment for container to be ready. +sleep 2 + +if ! docker ps --filter "name=$NAME" --format '{{.Names}}' | grep -q "$NAME"; then + echo "ERROR: Container failed to start" + tail -50 "$LOG" + exit 1 +fi + +# Record the container ID for later use. +docker inspect -f '{{.Id}}' "$NAME" > "$PID_FILE" + +echo "Container $NAME is running" +echo "Log: $LOG" diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh new file mode 100755 index 0000000..5753e46 --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Start SGLang server in Docker for a given TPxDP configuration. +# Usage: start_sglang_docker.sh +# +# Uses the lmsysorg/sglang image and the same argument set as the +# bare-metal start script. The container is removed automatically on stop. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" +PORT="${SGLANG_PORT:-30031}" +NAME="${EXPERIMENT}_sglang_tp${TP}_dp${DP}" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +# Clean up any stale container with the same name. +docker rm -f "$NAME" >/dev/null 2>&1 || true + +SERVER_ARGS=( + serve + --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$TP" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --dp-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting SGLang server in Docker (TP=${TP}, DP=${DP}) ===" +echo "Image: $IMAGE" +echo "Model: $MODEL_PATH" +echo "Container name: $NAME" +echo "Host port: $PORT" +echo "Command: sglang ${SERVER_ARGS_STR}" +echo "Log: $LOG" + +# Run docker in the foreground so that killing the host process stops the +# container (the --rm flag ensures cleanup). nohup lets us background it and +# capture the host PID in the same way as the bare-metal start script. +nohup docker run --rm \ + --name "$NAME" \ + --gpus all \ + --ipc host \ + --shm-size 16g \ + --entrypoint sglang \ + -p "${PORT}:${PORT}" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${RUNTIME_BASE}/tmp:/tmp" \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + "$IMAGE" \ + "${SERVER_ARGS[@]}" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: Docker SGLang server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: Docker SGLang server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh new file mode 100755 index 0000000..230bf88 --- /dev/null +++ b/experiments/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Start SGLang server for a given TP×DP configuration. +# Usage: start_sglang_dp.sh +# +# By default this delegates to the Docker start script because the experiment +# is intended to run SGLang inside a container. Set USE_DOCKER=0 to use the +# local VENV_CLIENT environment instead. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +if [[ "${USE_DOCKER:-1}" == "1" ]]; then + exec "${SCRIPT_DIR}/start_sglang_docker.sh" "$@" +fi + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +VENV="${VENV_CLIENT}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR="${RUNTIME_BASE}/tmp" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" + +rm -f "$PID_FILE" + +SERVER_ARGS=( + sglang serve + --model-path "$MODEL_PATH" + --trust-remote-code + --tp-size "$TP" + --moe-runner-backend "$MOE_RUNNER_BACKEND" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --host 0.0.0.0 + --port "$SGLANG_PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --dp-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting SGLang server (TP=${TP}, DP=${DP}) ===" +echo "Model: $MODEL_PATH" +echo "Port: $SGLANG_PORT" +echo "Command: $SERVER_ARGS_STR" +echo "Log: $LOG" + +nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${SGLANG_PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${SGLANG_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: SGLang server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: SGLang server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/platforms/nvidia_rtx6000d.env b/platforms/nvidia_rtx6000d.env new file mode 100644 index 0000000..5f720e9 --- /dev/null +++ b/platforms/nvidia_rtx6000d.env @@ -0,0 +1,23 @@ +# Platform configuration for NVIDIA RTX 6000D (Pro 6000D) +# Source this file via scripts/common/platform.sh + +CHIP="nvidia_rtx6000d" +ACCELERATOR="NVIDIA RTX 6000D" +HARDWARE="8x NVIDIA RTX 6000D 84GB" +ENGINE="sglang" + +# Device selection +DEVICE_SELECT_ENV="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7" +CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" + +# Default serving port and model root on the host +DEFAULT_PORT="30004" +MODEL_ROOT="/data/hf_models" + +# Virtual environments on the host (used by native scripts). +# Override these if your machine uses different paths. +VENV_SGLANG="${VENV_SGLANG:-/root/.miniconda3/envs/sglang}" +VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}" + +# Default server start script for the legacy benchmark grid. +SERVER_START_SCRIPT="${SERVER_START_SCRIPT:-${ROOT_DIR}/scripts/start_dsv4_dspark_8card.sh}"