diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/matrix.json b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/matrix.json index b7e45e3..dfc7572 100644 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/matrix.json +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/matrix.json @@ -1,7 +1,6 @@ { - "comment": "ISL/OSL matrix for dsv4_h20_sglang_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.", - "mode": "Y", "comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.", + "mode": "Y", "matrix": { "1024": { "128": "Y", @@ -19,6 +18,14 @@ "2048": "Y", "4096": "Y" }, + "8192": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, "16384": { "128": "Y", "256": "Y", @@ -27,6 +34,14 @@ "2048": "Y", "4096": "P" }, + "32768": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, "65536": { "128": "Y", "256": "Y", @@ -69,13 +84,45 @@ } }, "concurrency": { - "1024": { "low": 1, "high": 128 }, - "4096": { "low": 1, "high": 64 }, - "16384": { "low": 1, "high": 32 }, - "65536": { "low": 1, "high": 8 }, - "131072": { "low": 1, "high": 4 }, - "262144": { "low": 1, "high": 2 }, - "524288": { "low": 1, "high": 2 }, - "1048576": { "low": 1, "high": 2 } + "1024": { + "low": 1, + "high": 128 + }, + "4096": { + "low": 1, + "high": 64 + }, + "8192": { + "low": 1, + "high": 64 + }, + "16384": { + "low": 1, + "high": 32 + }, + "32768": { + "low": 1, + "high": 16 + }, + "65536": { + "low": 1, + "high": 8 + }, + "131072": { + "low": 1, + "high": 4 + }, + "262144": { + "low": 1, + "high": 2 + }, + "524288": { + "low": 1, + "high": 2 + }, + "1048576": { + "low": 1, + "high": 2 + } } } diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/matrix.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/matrix.json index 34e4847..dfc7572 100644 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/matrix.json +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/matrix.json @@ -1,7 +1,6 @@ { - "comment": "ISL/OSL matrix for dsv4_h20_vllm_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.", - "mode": "Y", "comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.", + "mode": "Y", "matrix": { "1024": { "128": "Y", @@ -19,6 +18,14 @@ "2048": "Y", "4096": "Y" }, + "8192": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, "16384": { "128": "Y", "256": "Y", @@ -27,6 +34,14 @@ "2048": "Y", "4096": "P" }, + "32768": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, "65536": { "128": "Y", "256": "Y", @@ -69,13 +84,45 @@ } }, "concurrency": { - "1024": { "low": 1, "high": 128 }, - "4096": { "low": 1, "high": 64 }, - "16384": { "low": 1, "high": 32 }, - "65536": { "low": 1, "high": 8 }, - "131072": { "low": 1, "high": 4 }, - "262144": { "low": 1, "high": 2 }, - "524288": { "low": 1, "high": 2 }, - "1048576": { "low": 1, "high": 2 } + "1024": { + "low": 1, + "high": 128 + }, + "4096": { + "low": 1, + "high": 64 + }, + "8192": { + "low": 1, + "high": 64 + }, + "16384": { + "low": 1, + "high": 32 + }, + "32768": { + "low": 1, + "high": 16 + }, + "65536": { + "low": 1, + "high": 8 + }, + "131072": { + "low": 1, + "high": 4 + }, + "262144": { + "low": 1, + "high": 2 + }, + "524288": { + "low": 1, + "high": 2 + }, + "1048576": { + "low": 1, + "high": 2 + } } } diff --git a/experiments/h200/dsv4_h200_sglang_tp_dp_matrix/matrix.json b/experiments/h200/dsv4_h200_sglang_tp_dp_matrix/matrix.json index 55dcf58..dfc7572 100644 --- a/experiments/h200/dsv4_h200_sglang_tp_dp_matrix/matrix.json +++ b/experiments/h200/dsv4_h200_sglang_tp_dp_matrix/matrix.json @@ -1,7 +1,6 @@ { - "comment": "ISL/DSL matrix for dsv4_h200_sglang_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.", - "mode": "Y", "comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.", + "mode": "Y", "matrix": { "1024": { "128": "Y", @@ -19,6 +18,14 @@ "2048": "Y", "4096": "Y" }, + "8192": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, "16384": { "128": "Y", "256": "Y", @@ -27,6 +34,14 @@ "2048": "Y", "4096": "P" }, + "32768": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, "65536": { "128": "Y", "256": "Y", @@ -69,13 +84,45 @@ } }, "concurrency": { - "1024": { "low": 1, "high": 128 }, - "4096": { "low": 1, "high": 64 }, - "16384": { "low": 1, "high": 32 }, - "65536": { "low": 1, "high": 8 }, - "131072": { "low": 1, "high": 4 }, - "262144": { "low": 1, "high": 2 }, - "524288": { "low": 1, "high": 2 }, - "1048576": { "low": 1, "high": 2 } + "1024": { + "low": 1, + "high": 128 + }, + "4096": { + "low": 1, + "high": 64 + }, + "8192": { + "low": 1, + "high": 64 + }, + "16384": { + "low": 1, + "high": 32 + }, + "32768": { + "low": 1, + "high": 16 + }, + "65536": { + "low": 1, + "high": 8 + }, + "131072": { + "low": 1, + "high": 4 + }, + "262144": { + "low": 1, + "high": 2 + }, + "524288": { + "low": 1, + "high": 2 + }, + "1048576": { + "low": 1, + "high": 2 + } } } diff --git a/experiments/h200/dsv4_h200_vllm_tp_dp_matrix/matrix.json b/experiments/h200/dsv4_h200_vllm_tp_dp_matrix/matrix.json index 539a577..dfc7572 100644 --- a/experiments/h200/dsv4_h200_vllm_tp_dp_matrix/matrix.json +++ b/experiments/h200/dsv4_h200_vllm_tp_dp_matrix/matrix.json @@ -1,7 +1,6 @@ { - "comment": "ISL/DSL matrix for dsv4_h200_vllm_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.", - "mode": "Y", "comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.", + "mode": "Y", "matrix": { "1024": { "128": "Y", @@ -19,6 +18,14 @@ "2048": "Y", "4096": "Y" }, + "8192": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, "16384": { "128": "Y", "256": "Y", @@ -27,6 +34,14 @@ "2048": "Y", "4096": "P" }, + "32768": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, "65536": { "128": "Y", "256": "Y", @@ -69,13 +84,45 @@ } }, "concurrency": { - "1024": { "low": 1, "high": 128 }, - "4096": { "low": 1, "high": 64 }, - "16384": { "low": 1, "high": 32 }, - "65536": { "low": 1, "high": 8 }, - "131072": { "low": 1, "high": 4 }, - "262144": { "low": 1, "high": 2 }, - "524288": { "low": 1, "high": 2 }, - "1048576": { "low": 1, "high": 2 } + "1024": { + "low": 1, + "high": 128 + }, + "4096": { + "low": 1, + "high": 64 + }, + "8192": { + "low": 1, + "high": 64 + }, + "16384": { + "low": 1, + "high": 32 + }, + "32768": { + "low": 1, + "high": 16 + }, + "65536": { + "low": 1, + "high": 8 + }, + "131072": { + "low": 1, + "high": 4 + }, + "262144": { + "low": 1, + "high": 2 + }, + "524288": { + "low": 1, + "high": 2 + }, + "1048576": { + "low": 1, + "high": 2 + } } } diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/matrix.json b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/matrix.json index a0b8099..48febc3 100644 --- a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/matrix.json +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/matrix.json @@ -19,6 +19,14 @@ "2048": "Y", "4096": "Y" }, + "8192": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, "16384": { "128": "Y", "256": "Y", @@ -27,6 +35,14 @@ "2048": "Y", "4096": "P" }, + "32768": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, "65536": { "128": "Y", "256": "Y", @@ -45,10 +61,33 @@ } }, "concurrency": { - "1024": { "low": 1, "high": 128 }, - "4096": { "low": 1, "high": 64 }, - "16384": { "low": 1, "high": 32 }, - "65536": { "low": 1, "high": 8 }, - "131072": { "low": 1, "high": 4 } + "1024": { + "low": 1, + "high": 128 + }, + "4096": { + "low": 1, + "high": 64 + }, + "8192": { + "low": 1, + "high": 64 + }, + "16384": { + "low": 1, + "high": 32 + }, + "32768": { + "low": 1, + "high": 16 + }, + "65536": { + "low": 1, + "high": 8 + }, + "131072": { + "low": 1, + "high": 4 + } } } diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env index 1eda1f0..204f531 100644 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env @@ -15,11 +15,16 @@ NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" +# Stop a shape when p95 TTFT exceeds the SLO; keep group skipping disabled. +TTFT_SLO_MS="${TTFT_SLO_MS:-4000}" +ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}" +TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-0}" + # Keep the same random workload semantics as the fixed matrix baseline. # DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times # NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to # random-ids to use generated token IDs without a ShareGPT seed dataset. -BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random-ids}" # SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed # ISL/OSL points; lower values intentionally benchmark a length distribution. RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/compare.py b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/compare.py index 2029e7b..981609a 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/compare.py +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/compare.py @@ -83,7 +83,7 @@ def main(): f.write(f"- Model: `{model}`\n") f.write(f"- Hardware: {hardware}\n") f.write("- Backend: SGLang (Docker)\n") - f.write("- Benchmark client: `sglang.bench_serving`\n") + f.write("- Benchmark client: `sglang.benchmark.serving`\n") f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n") # Configuration overview. diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env index 9521ce9..596a0aa 100644 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env @@ -7,7 +7,7 @@ EXPERIMENT="dsv4_pro6000_sglang_tp_dp_matrix" MODEL_NAME="DeepSeek-V4-Flash" -MODEL_PATH="/root/data/models/ds_flash_v4" +MODEL_PATH="/data/6000D/DeepSeek-V4-Flash" SERVED_MODEL_NAME="deepseek-v4-flash" SGLANG_PORT="${SGLANG_PORT:-30031}" @@ -20,7 +20,7 @@ VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}" # Run the benchmark client natively (0) or inside Docker (1). USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" -export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,3,4,5,6,7}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" # Runtime working directory for logs, pid files, and tmp. Defaults to a local # directory under this experiment so the benchmark is self-contained. @@ -41,12 +41,14 @@ MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}" # Deployment switch. 0 = native sglang venv, 1 = Docker. USE_DOCKER="${USE_DOCKER:-1}" -DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" +DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}" -# Dataset used by sglang.bench_serving --dataset-name random. -# The random sampler needs a ShareGPT-style JSON file locally; it falls back to -# downloading from HuggingFace, which usually fails on offline nodes. -DATASET_PATH="${DATASET_PATH:-/root/data/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}" +# Offline by default: random-ids generates token IDs without downloading data. +# To use ShareGPT, set BENCH_DATASET_NAME=random and DATASET_PATH explicitly. +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random-ids}" +DATASET_PATH="${DATASET_PATH:-}" +SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}" +CACHE_DIR="${CACHE_DIR:-/data/yy/sskj/sglang_sm120_cache}" # Matrix and concurrency rules are defined in matrix.json by default. MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/matrix.json b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/matrix.json index 55dcf58..dfc7572 100644 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/matrix.json +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/matrix.json @@ -1,7 +1,6 @@ { - "comment": "ISL/DSL matrix for dsv4_h200_sglang_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.", - "mode": "Y", "comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.", + "mode": "Y", "matrix": { "1024": { "128": "Y", @@ -19,6 +18,14 @@ "2048": "Y", "4096": "Y" }, + "8192": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, "16384": { "128": "Y", "256": "Y", @@ -27,6 +34,14 @@ "2048": "Y", "4096": "P" }, + "32768": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, "65536": { "128": "Y", "256": "Y", @@ -69,13 +84,45 @@ } }, "concurrency": { - "1024": { "low": 1, "high": 128 }, - "4096": { "low": 1, "high": 64 }, - "16384": { "low": 1, "high": 32 }, - "65536": { "low": 1, "high": 8 }, - "131072": { "low": 1, "high": 4 }, - "262144": { "low": 1, "high": 2 }, - "524288": { "low": 1, "high": 2 }, - "1048576": { "low": 1, "high": 2 } + "1024": { + "low": 1, + "high": 128 + }, + "4096": { + "low": 1, + "high": 64 + }, + "8192": { + "low": 1, + "high": 64 + }, + "16384": { + "low": 1, + "high": 32 + }, + "32768": { + "low": 1, + "high": 16 + }, + "65536": { + "low": 1, + "high": 8 + }, + "131072": { + "low": 1, + "high": 4 + }, + "262144": { + "low": 1, + "high": 2 + }, + "524288": { + "low": 1, + "high": 2 + }, + "1048576": { + "low": 1, + "high": 2 + } } } diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh index 7972a9e..5952558 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh @@ -27,7 +27,7 @@ else PYTHON="$(command -v python3)" fi -DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" +DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}" engine_is_healthy() { curl --fail --silent --show-error --max-time 5 \ @@ -68,7 +68,7 @@ engine_build_server_args() { local tp="$1" local dp="$2" local -a args=( - sglang serve --model-path "$MODEL_PATH" + python3 -m sglang.launch_server --model-path "$MODEL_PATH" --trust-remote-code --tp-size "$tp" --moe-runner-backend "$MOE_RUNNER_BACKEND" @@ -167,15 +167,16 @@ engine_run_bench() { --network host \ "${volume_args[@]}" \ -e PYTHONUNBUFFERED=1 \ + --entrypoint python3 \ "$DOCKER_IMAGE" \ - python -m sglang.bench_serving "${bench_args[@]}" + -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}" else - "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + "$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}" fi } export -f engine_run_bench export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT -export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE adaptive_main "$@" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh index c98b531..de33dc6 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -27,7 +27,7 @@ else PYTHON="$(command -v python3)" fi -DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" +DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}" engine_is_healthy() { curl --fail --silent --show-error --max-time 5 \ @@ -68,7 +68,7 @@ engine_build_server_args() { local tp="$1" local dp="$2" local -a args=( - sglang serve --model-path "$MODEL_PATH" + python3 -m sglang.launch_server --model-path "$MODEL_PATH" --trust-remote-code --tp-size "$tp" --moe-runner-backend "$MOE_RUNNER_BACKEND" @@ -167,16 +167,17 @@ engine_run_bench() { --network host \ "${volume_args[@]}" \ -e PYTHONUNBUFFERED=1 \ + --entrypoint python3 \ "$DOCKER_IMAGE" \ - python -m sglang.bench_serving "${bench_args[@]}" + -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}" else - "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + "$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}" fi } export -f engine_run_bench export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT -export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE export SEARCH_START_CONCURRENCY=16 export SEARCH_ADDEND=16 diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh index a8bc490..d2b9c84 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh @@ -22,7 +22,7 @@ DRY_RUN="${DRY_RUN:-0}" GRID_LIMIT="${GRID_LIMIT:-0}" # Export variables used inside functions that are called via bash -c subshells. -export DATASET_PATH MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT SGLANG_BENCH_MODULE if [[ -x "${VENV_CLIENT}/bin/python" ]]; then PYTHON="${VENV_CLIENT}/bin/python" @@ -30,7 +30,7 @@ else PYTHON="$(command -v python3)" fi -DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" +DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}" log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" mkdir -p "$log_dir_global" @@ -79,9 +79,9 @@ stop_server() { docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true fi - # Fallback: kill any sglang serve processes for this model. - pkill -9 -f "sglang serve.*${MODEL_NAME}" 2>/dev/null || true - pkill -9 -f "sglang serve.*${MODEL_PATH}" 2>/dev/null || true + # Fallback: kill any SGLang launch processes for this model. + pkill -9 -f "sglang.launch_server.*${MODEL_NAME}" 2>/dev/null || true + pkill -9 -f "sglang.launch_server.*${MODEL_PATH}" 2>/dev/null || true sleep 2 } @@ -90,7 +90,7 @@ build_server_args() { local dp="$2" local args=( - "sglang serve" "$MODEL_PATH" + "python3 -m sglang.launch_server" --model-path "$MODEL_PATH" --trust-remote-code --tp-size "$tp" --moe-runner-backend "$MOE_RUNNER_BACKEND" @@ -138,11 +138,17 @@ restart_server() { } run_bench_serving() { - # Run sglang.bench_serving either natively or inside the SGLang Docker image. + # Inject the offline workload choice consistently for warmup and main runs. + local -a dataset_args=(--dataset-name "$BENCH_DATASET_NAME") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + dataset_args+=(--dataset-path "$DATASET_PATH") + else + dataset_args+=(--tokenize-prompt) + fi if [[ "${USE_DOCKER_CLIENT:-1}" == "1" ]]; then local vol_args=() vol_args+=("-v" "${MODEL_PATH}:${MODEL_PATH}:ro") - if [[ -n "${DATASET_PATH:-}" ]]; then + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then vol_args+=("-v" "${DATASET_PATH}:${DATASET_PATH}:ro") fi vol_args+=("-v" "${RESULT_BASE}:${RESULT_BASE}") @@ -150,10 +156,11 @@ run_bench_serving() { --network host \ "${vol_args[@]}" \ -e PYTHONUNBUFFERED=1 \ + --entrypoint python3 \ "${DOCKER_IMAGE}" \ - python -m sglang.bench_serving "$@" + -m "$SGLANG_BENCH_MODULE" "${dataset_args[@]}" "$@" else - "$PYTHON" -m sglang.bench_serving "$@" + "$PYTHON" -m "$SGLANG_BENCH_MODULE" "${dataset_args[@]}" "$@" fi } export -f run_bench_serving @@ -168,8 +175,6 @@ run_warmup() { --backend sglang \ --host 127.0.0.1 \ --port "'"$SGLANG_PORT"'" \ - --dataset-name random \ - --dataset-path "'"$DATASET_PATH"'" \ --random-input-len "'"$input_len"'" \ --random-output-len "'"$output_len"'" \ --num-prompts 1 \ @@ -319,7 +324,7 @@ skip_remaining_scenarios() { fi append_scenario_record "$result_root" \ "name=${sname}" \ - "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ "status=\"${status}\"" \ "note=\"${reason}\"" record_skipped_csv "$skipped_csv" \ @@ -447,8 +452,6 @@ run_parallel_config() { --backend sglang \ --host 127.0.0.1 \ --port "'"$SGLANG_PORT"'" \ - --dataset-name random \ - --dataset-path "'"$DATASET_PATH"'" \ --random-input-len "'"$isl"'" \ --random-output-len "'"$dsl"'" \ --num-prompts "'"$num"'" \ @@ -465,7 +468,7 @@ run_parallel_config() { log "finished ${config_label} scenario: output=${output_file}" append_scenario_record "$result_root" \ "name=${sname}" \ - "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ "status=\"completed\"" \ "note=\"benchmark finished successfully\"" continue @@ -476,7 +479,7 @@ run_parallel_config() { log "ERROR: ${config_label} scenario ${sname} triggered OOM; stopping config" append_scenario_record "$result_root" \ "name=${sname}" \ - "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ "status=\"OOM\"" \ "note=\"detected CUDA out-of-memory\"" record_skipped_csv "$skipped_csv" \ @@ -491,7 +494,7 @@ run_parallel_config() { log "optional (P) scenario failed; recording as skipped and continuing" append_scenario_record "$result_root" \ "name=${sname}" \ - "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ "status=\"skipped_optional\"" \ "note=\"optional scenario failed (rc=${bench_rc})\"" record_skipped_csv "$skipped_csv" \ @@ -509,7 +512,7 @@ run_parallel_config() { log "ERROR: ${config_label} server restart failed; skipping remaining scenarios" append_scenario_record "$result_root" \ "name=${sname}" \ - "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ "status=\"FAILED\"" \ "note=\"scenario failed and server restart failed (rc=${bench_rc})\"" record_skipped_csv "$skipped_csv" \ diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh index 69a08b0..5346b6b 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh @@ -22,7 +22,7 @@ rm -f "$PID_FILE" # Build server args. SERVER_ARGS=( - serve + -m sglang.launch_server --model-path "$MODEL_PATH" --trust-remote-code --tp-size "$TP" @@ -44,24 +44,24 @@ SERVER_ARGS_STR="${SERVER_ARGS[*]}" echo "=== Starting SGLang server inside container (TP=${TP}, DP=${DP}) ===" echo "Container: $NAME" -echo "Command: sglang ${SERVER_ARGS_STR}" +echo "Command: python3 ${SERVER_ARGS_STR}" echo "Log: $LOG" # Kill any existing sglang process inside container first. -docker exec "$NAME" pkill -9 -f "sglang serve" 2>/dev/null || true +docker exec "$NAME" pkill -9 -f "sglang.launch_server" 2>/dev/null || true sleep 3 -# Start sglang serve inside the container in background. +# Start the SGLang launch module inside the container in background. # We use nohup so it survives after docker exec returns. docker exec -d \ -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ -e PYTHONUNBUFFERED=1 \ "$NAME" \ - bash -c "nohup sglang ${SERVER_ARGS_STR} > /tmp/sglang_server.log 2>&1 &" + bash -c "nohup python3 ${SERVER_ARGS_STR} > /tmp/sglang_server.log 2>&1 &" # Wait for the server process to appear inside container. sleep 2 -SERVER_PID=$(docker exec "$NAME" pgrep -f "sglang serve" | head -n 1 || true) +SERVER_PID=$(docker exec "$NAME" pgrep -f "sglang.launch_server" | head -n 1 || true) if [[ -z "$SERVER_PID" ]]; then echo "ERROR: sglang server process not found inside container" docker exec "$NAME" cat /tmp/sglang_server.log 2>/dev/null | tail -50 || true diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_container.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_container.sh index 2cf469c..c33bb0d 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_container.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_container.sh @@ -13,14 +13,13 @@ source "${SCRIPT_DIR}/config.env" RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" -IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" +IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}" PORT="${SGLANG_PORT:-30031}" NAME="${EXPERIMENT}_sglang_container" PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_container.pid" -# Persistent cache directory on host for DeepGEMM JIT kernels. -CACHE_DIR="${RUNTIME_BASE}/cache" -mkdir -p "${CACHE_DIR}/deep_gemm" "${CACHE_DIR}/tvm-ffi" +# Persistent cache directory for SM120 JIT kernels and tokenizer metadata. +mkdir -p "$CACHE_DIR" # Clean up any stale container. docker rm -f "$NAME" >/dev/null 2>&1 || true @@ -40,16 +39,19 @@ echo "Log: $LOG" docker run -d \ --name "$NAME" \ --gpus all \ - --ipc host \ - --shm-size 16g \ + --privileged \ + --ipc=host \ + --network host \ + --ulimit memlock=-1 \ + --ulimit stack=67108864 \ --entrypoint /bin/bash \ - -p "${PORT}:${PORT}" \ -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ - -v "${CACHE_DIR}/deep_gemm:/root/.cache/deep_gemm" \ - -v "${CACHE_DIR}/tvm-ffi:/root/.cache/tvm-ffi" \ + -v "${CACHE_DIR}:/root/.cache" \ -v "${RUNTIME_BASE}/tmp:/tmp" \ -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ "$IMAGE" \ -c "sleep infinity" \ > "$LOG" 2>&1 diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh index 5753e46..29f27fe 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh @@ -2,8 +2,8 @@ # Start SGLang server in Docker for a given TPxDP configuration. # Usage: start_sglang_docker.sh # -# Uses the lmsysorg/sglang image and the same argument set as the -# bare-metal start script. The container is removed automatically on stop. +# Uses the verified SGLang SM120 image and keeps its JIT cache on +# persistent storage. The container is removed automatically on stop. set -e TP="${1}" @@ -14,9 +14,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${SCRIPT_DIR}/config.env" RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" -mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" "$CACHE_DIR" -IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}" +IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}" PORT="${SGLANG_PORT:-30031}" NAME="${EXPERIMENT}_sglang_tp${TP}_dp${DP}" PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" @@ -28,7 +28,7 @@ rm -f "$PID_FILE" docker rm -f "$NAME" >/dev/null 2>&1 || true SERVER_ARGS=( - serve + -m sglang.launch_server --model-path "$MODEL_PATH" --trust-remote-code --tp-size "$TP" @@ -53,7 +53,7 @@ echo "Image: $IMAGE" echo "Model: $MODEL_PATH" echo "Container name: $NAME" echo "Host port: $PORT" -echo "Command: sglang ${SERVER_ARGS_STR}" +echo "Command: python3 ${SERVER_ARGS_STR}" echo "Log: $LOG" # Run docker in the foreground so that killing the host process stops the @@ -62,14 +62,19 @@ echo "Log: $LOG" nohup docker run --rm \ --name "$NAME" \ --gpus all \ - --ipc host \ - --shm-size 16g \ - --entrypoint sglang \ - -p "${PORT}:${PORT}" \ + --privileged \ + --ipc=host \ + --network host \ + --ulimit memlock=-1 \ + --ulimit stack=67108864 \ + --entrypoint python3 \ -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${CACHE_DIR}:/root/.cache" \ -v "${RUNTIME_BASE}/tmp:/tmp" \ -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ "$IMAGE" \ "${SERVER_ARGS[@]}" \ > "$LOG" 2>&1 & diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh index 230bf88..3472823 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh @@ -33,7 +33,7 @@ PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid" rm -f "$PID_FILE" SERVER_ARGS=( - sglang serve + python3 -m sglang.launch_server --model-path "$MODEL_PATH" --trust-remote-code --tp-size "$TP" diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env index 1eda1f0..204f531 100644 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env @@ -15,11 +15,16 @@ NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" +# Stop a shape when p95 TTFT exceeds the SLO; keep group skipping disabled. +TTFT_SLO_MS="${TTFT_SLO_MS:-4000}" +ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}" +TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-0}" + # Keep the same random workload semantics as the fixed matrix baseline. # DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times # NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to # random-ids to use generated token IDs without a ShareGPT seed dataset. -BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random-ids}" # SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed # ISL/OSL points; lower values intentionally benchmark a length distribution. RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/config.env b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/config.env index 228a6e4..1d37f9b 100644 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/config.env +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/config.env @@ -6,7 +6,7 @@ EXPERIMENT="dsv4_pro6000_vllm_tp_dp_matrix" MODEL_NAME="DeepSeek-V4-Flash" -MODEL_PATH="/data/hf_models/DeepSeek-V4-Flash" +MODEL_PATH="/data/6000D/DeepSeek-V4-Flash" SERVED_MODEL_NAME="deepseek-v4-flash" VLLM_PORT="${VLLM_PORT:-30030}" @@ -41,17 +41,19 @@ BLOCK_SIZE="${BLOCK_SIZE:-256}" # Deployment switch. 0 = native vllm venv, 1 = Docker. USE_DOCKER="${USE_DOCKER:-1}" -DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}" -# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving, +# Benchmark client Docker image. vLLM's image does not include the SGLang benchmark client, # so the client runs inside the SGLang image and targets the vLLM backend via the # OpenAI-compatible API. -DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}" -# Dataset used by sglang.bench_serving --dataset-name random. -# The random sampler needs a ShareGPT-style JSON file locally; it falls back to -# downloading from HuggingFace, which usually fails on offline nodes. -DATASET_PATH="${DATASET_PATH:-/data/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}" +# Offline by default: random-ids generates token IDs without downloading data. +# To use ShareGPT, set BENCH_DATASET_NAME=random and DATASET_PATH explicitly. +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random-ids}" +DATASET_PATH="${DATASET_PATH:-}" +SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}" +CACHE_DIR="${CACHE_DIR:-/data/yy/sskj/vllm_sm120_cache}" # Matrix and concurrency rules are defined in matrix.json by default. MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/matrix.json b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/matrix.json new file mode 100644 index 0000000..1cc6712 --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/matrix.json @@ -0,0 +1,128 @@ +{ + "comment": "ISL/OSL matrix for dsv4_pro6000_vllm_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.", + "mode": "Y", + "matrix": { + "1024": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "4096": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "8192": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "16384": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, + "32768": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, + "65536": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "P", + "4096": "N" + }, + "131072": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "P", + "2048": "N", + "4096": "N" + }, + "262144": { + "128": "Y", + "256": "Y", + "512": "P", + "1024": "N", + "2048": "N", + "4096": "N" + }, + "524288": { + "128": "Y", + "256": "P", + "512": "N", + "1024": "N", + "2048": "N", + "4096": "N" + }, + "1048576": { + "128": "Y", + "256": "P", + "512": "N", + "1024": "N", + "2048": "N", + "4096": "N" + } + }, + "concurrency": { + "1024": { + "low": 1, + "high": 128 + }, + "4096": { + "low": 1, + "high": 64 + }, + "8192": { + "low": 1, + "high": 64 + }, + "16384": { + "low": 1, + "high": 32 + }, + "32768": { + "low": 1, + "high": 16 + }, + "65536": { + "low": 1, + "high": 8 + }, + "131072": { + "low": 1, + "high": 4 + }, + "262144": { + "low": 1, + "high": 2 + }, + "524288": { + "low": 1, + "high": 2 + }, + "1048576": { + "low": 1, + "high": 2 + } + } +} diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh index f76a93d..1e0c315 100755 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh @@ -27,8 +27,8 @@ else PYTHON="$(command -v python3)" fi -DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" -DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}" engine_is_healthy() { curl --fail --silent --show-error --max-time 5 \ @@ -68,7 +68,6 @@ engine_build_server_args() { --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" --max-model-len "$MAX_MODEL_LEN" --max-num-seqs "$MAX_NUM_SEQS" - --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$ENGINE_PORT" ) @@ -150,15 +149,16 @@ engine_run_bench() { -e PYTHONUNBUFFERED=1 \ -e HF_HUB_OFFLINE=1 \ -e TRANSFORMERS_OFFLINE=1 \ + --entrypoint python3 \ "$DOCKER_CLIENT_IMAGE" \ - python -m sglang.bench_serving "${bench_args[@]}" + -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}" else - "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + "$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}" fi } export -f engine_run_bench export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT -export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE adaptive_main "$@" diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh index fb18812..aa1c47b 100755 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -27,8 +27,8 @@ else PYTHON="$(command -v python3)" fi -DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" -DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}" engine_is_healthy() { curl --fail --silent --show-error --max-time 5 \ @@ -68,7 +68,6 @@ engine_build_server_args() { --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" --max-model-len "$MAX_MODEL_LEN" --max-num-seqs "$MAX_NUM_SEQS" - --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$ENGINE_PORT" ) @@ -150,16 +149,17 @@ engine_run_bench() { -e PYTHONUNBUFFERED=1 \ -e HF_HUB_OFFLINE=1 \ -e TRANSFORMERS_OFFLINE=1 \ + --entrypoint python3 \ "$DOCKER_CLIENT_IMAGE" \ - python -m sglang.bench_serving "${bench_args[@]}" + -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}" else - "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + "$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}" fi } export -f engine_run_bench export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT -export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE export SEARCH_START_CONCURRENCY=16 export SEARCH_ADDEND=16 diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh index e133fd7..14807a8 100755 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -2,8 +2,8 @@ # Start vLLM server in Docker for a given TPxDP configuration. # Usage: start_vllm_docker.sh # -# Uses the vllm/vllm-openai image and the same argument set as the -# bare-metal start script. The container is removed automatically on stop. +# Uses the verified vLLM SM120 image and keeps its cache on persistent +# storage. The container is removed automatically on stop. set -e TP="${1}" @@ -14,9 +14,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${SCRIPT_DIR}/config.env" RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" -mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" "$CACHE_DIR" -IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}" PORT="${VLLM_PORT:-30030}" NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}" PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" @@ -36,7 +36,6 @@ SERVER_ARGS=( --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" --max-model-len "$MAX_MODEL_LEN" --max-num-seqs "$MAX_NUM_SEQS" - --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$PORT" ) @@ -63,12 +62,14 @@ echo "Log: $LOG" nohup docker run --rm \ --name "$NAME" \ --gpus all \ - --ipc host \ - --shm-size 16g \ + --privileged \ + --ipc=host \ + --network host \ --ulimit memlock=-1 \ + --ulimit stack=67108864 \ --entrypoint vllm \ - -p "${PORT}:${PORT}" \ -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${CACHE_DIR}:/root/.cache" \ -v "${RUNTIME_BASE}/tmp:/tmp" \ -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ -e PYTHONUNBUFFERED=1 \ diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh index 741ca9c..b8c0b02 100755 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -41,7 +41,6 @@ SERVER_ARGS=( --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" --max-model-len "$MAX_MODEL_LEN" --max-num-seqs "$MAX_NUM_SEQS" - --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$VLLM_PORT" ) diff --git a/scripts/common/platform.sh b/scripts/common/platform.sh index 8fea032..77c4f4f 100755 --- a/scripts/common/platform.sh +++ b/scripts/common/platform.sh @@ -21,7 +21,9 @@ if [[ -z "${PLATFORM:-}" ]]; then PLATFORM="kunlun_p800" elif command -v nvidia-smi >/dev/null 2>&1; then _GPU_NAME=$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -n 1 || true) - if [[ -n "$_GPU_NAME" ]] && grep -qi "H20" <<< "$_GPU_NAME"; then + if [[ -n "$_GPU_NAME" ]] && grep -qiE "RTX 6000D|PRO 6000D" <<< "$_GPU_NAME"; then + PLATFORM="nvidia_rtx6000d" + elif [[ -n "$_GPU_NAME" ]] && grep -qi "H20" <<< "$_GPU_NAME"; then PLATFORM="nvidia_h20" else PLATFORM="nvidia_h200"