From 7852dd20404e4962bbc321788f387c383a9f2d43 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Fri, 10 Jul 2026 02:17:00 +0000 Subject: [PATCH] sglang tp_dp_matrix: run SGLang server/client in Docker on H200 - config.env: switch to /data/models/DeepSeek-V4-Flash, USE_DOCKER=1, CONTEXT_LENGTH=1M, CONCURRENCY_SAMPLES=2, add DATASET_PATH. - start_sglang_docker.sh: add --ipc host --shm-size 16g for NCCL. - start_sglang_dp.sh: use --model-path for Docker CLI. - run_bench.sh: mount dataset into client container and pass --dataset-path to avoid HF download on offline nodes. --- .../dsv4_h200_sglang_tp_dp_matrix/config.env | 26 +++++++++++-------- .../run_bench.sh | 3 +++ .../start_sglang_docker.sh | 5 +++- .../start_sglang_dp.sh | 3 ++- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env b/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env index 600b4cd..d5ac289 100644 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/config.env @@ -1,4 +1,4 @@ -# TP×DP matrix experiment for DeepSeek-V4-Flash on H200 (8 GPUs) using SGLang Docker. +# TP×DP matrix experiment for DeepSeek-V4-Flash on H200 (8 GPUs) using SGLang. # Tests SGLang with three parallel configurations: # TP=2, DP=4 -> 2 GPUs per replica, 4 replicas # TP=4, DP=2 -> 4 GPUs per replica, 2 replicas @@ -6,7 +6,7 @@ EXPERIMENT="dsv4_h200_sglang_tp_dp_matrix" MODEL_NAME="DeepSeek-V4-Flash" -MODEL_PATH="/data3/hf_models/DeepSeek-V4-Flash" +MODEL_PATH="/data/models/DeepSeek-V4-Flash" SERVED_MODEL_NAME="deepseek-v4-flash" SGLANG_PORT="${SGLANG_PORT:-30031}" @@ -16,9 +16,7 @@ SGLANG_PORT="${SGLANG_PORT:-30031}" # does not exist on the host. VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}" -# When USE_DOCKER_CLIENT=1, the benchmark client (sglang.bench_serving) is run -# inside the same SGLang Docker image used for the server. This avoids requiring -# a native sglang installation on the host. +# Run the benchmark client natively (0) or inside Docker (1). USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" @@ -35,22 +33,28 @@ declare -a PARALLEL_CONFIGS=( ) # SGLang server settings -CONTEXT_LENGTH="${CONTEXT_LENGTH:-65536}" -MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}" +CONTEXT_LENGTH="${CONTEXT_LENGTH:-1048576}" +MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-128}" MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.88}" MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}" -# Docker deployment switch. SGLang is launched via Docker using the image below. +# Deployment switch. 0 = native sglang venv, 1 = Docker. USE_DOCKER="${USE_DOCKER:-1}" DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" +# Dataset used by sglang.bench_serving --dataset-name random. +# The random sampler needs a ShareGPT-style JSON file locally; it falls back to +# downloading from HuggingFace, which usually fails on offline H200 nodes. +DATASET_PATH="${DATASET_PATH:-/data/user1/yy/datasets/ShareGPT_V4.3_unfiltered_cleaned_split.json}" + # Matrix and concurrency rules are defined in matrix.json by default. MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" MATRIX_MODE="${MATRIX_MODE:-Y}" -# Sampling density for concurrency. 0 means use the default heuristic in -# generate_scenarios.py (6-8 points, or all integers when range is small). -CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-0}" +# Sampling density for concurrency. +# 0 = use the default heuristic in generate_scenarios.py (6-8 points). +# 2 = only test the low and high endpoints. +export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}" # Per-scenario timeout to avoid hangs (seconds). SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh index 75713d0..09a5e09 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_bench.sh @@ -123,6 +123,7 @@ run_bench_serving() { docker run --rm \ --network host \ -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${DATASET_PATH}:${DATASET_PATH}:ro" \ -v "${RESULT_BASE}:${RESULT_BASE}" \ -e PYTHONUNBUFFERED=1 \ "${DOCKER_IMAGE}" \ @@ -142,6 +143,7 @@ run_warmup() { --host 127.0.0.1 \ --port "$SGLANG_PORT" \ --dataset-name random \ + --dataset-path "$DATASET_PATH" \ --random-input-len "$input_len" \ --random-output-len "$output_len" \ --num-prompts 1 \ @@ -419,6 +421,7 @@ run_parallel_config() { --host 127.0.0.1 \ --port "$SGLANG_PORT" \ --dataset-name random \ + --dataset-path "$DATASET_PATH" \ --random-input-len "$isl" \ --random-output-len "$dsl" \ --num-prompts "$num" \ diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_docker.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_docker.sh index 4ddd72f..5753e46 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_docker.sh +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_docker.sh @@ -28,7 +28,8 @@ rm -f "$PID_FILE" docker rm -f "$NAME" >/dev/null 2>&1 || true SERVER_ARGS=( - serve "$MODEL_PATH" + serve + --model-path "$MODEL_PATH" --trust-remote-code --tp-size "$TP" --moe-runner-backend "$MOE_RUNNER_BACKEND" @@ -61,6 +62,8 @@ echo "Log: $LOG" nohup docker run --rm \ --name "$NAME" \ --gpus all \ + --ipc host \ + --shm-size 16g \ --entrypoint sglang \ -p "${PORT}:${PORT}" \ -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_dp.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_dp.sh index 35d52fe..230bf88 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_dp.sh +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/start_sglang_dp.sh @@ -33,7 +33,8 @@ PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" rm -f "$PID_FILE" SERVER_ARGS=( - sglang serve "$MODEL_PATH" + sglang serve + --model-path "$MODEL_PATH" --trust-remote-code --tp-size "$TP" --moe-runner-backend "$MOE_RUNNER_BACKEND"