qqtang/sskj
1
2
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update 8k and 32k

Browse Source
This commit is contained in:
Quantong Qiu 2026-07-17 16:47:25 +08:00
parent b3ead2cb77
commit 8beed2411f
24 changed files with 580 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
experiments/h20/dsv4_h20_sglang_tp_dp_matrix/matrix.json
View File

@ -1,7 +1,6 @@
{
"comment": "ISL/OSL matrix for dsv4_h20_sglang_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.",
"mode": "Y",
"comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.",
"mode": "Y",
"matrix": {
"1024": {
"128": "Y",
@ -19,6 +18,14 @@
"2048": "Y",
"4096": "Y"
},
"8192": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "Y"
},
"16384": {
"128": "Y",
"256": "Y",
@ -27,6 +34,14 @@
"2048": "Y",
"4096": "P"
},
"32768": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "P"
},
"65536": {
"128": "Y",
"256": "Y",
@ -69,13 +84,45 @@
}
},
"concurrency": {
"1024": { "low": 1, "high": 128 },
"4096": { "low": 1, "high": 64 },
"16384": { "low": 1, "high": 32 },
"65536": { "low": 1, "high": 8 },
"131072": { "low": 1, "high": 4 },
"262144": { "low": 1, "high": 2 },
"524288": { "low": 1, "high": 2 },
"1048576": { "low": 1, "high": 2 }
"1024": {
"low": 1,
"high": 128
},
"4096": {
"low": 1,
"high": 64
},
"8192": {
"low": 1,
"high": 64
},
"16384": {
"low": 1,
"high": 32
},
"32768": {
"low": 1,
"high": 16
},
"65536": {
"low": 1,
"high": 8
},
"131072": {
"low": 1,
"high": 4
},
"262144": {
"low": 1,
"high": 2
},
"524288": {
"low": 1,
"high": 2
},
"1048576": {
"low": 1,
"high": 2
}
}
}

67
experiments/h20/dsv4_h20_vllm_tp_dp_matrix/matrix.json
View File

@ -1,7 +1,6 @@
{
"comment": "ISL/OSL matrix for dsv4_h20_vllm_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.",
"mode": "Y",
"comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.",
"mode": "Y",
"matrix": {
"1024": {
"128": "Y",
@ -19,6 +18,14 @@
"2048": "Y",
"4096": "Y"
},
"8192": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "Y"
},
"16384": {
"128": "Y",
"256": "Y",
@ -27,6 +34,14 @@
"2048": "Y",
"4096": "P"
},
"32768": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "P"
},
"65536": {
"128": "Y",
"256": "Y",
@ -69,13 +84,45 @@
}
},
"concurrency": {
"1024": { "low": 1, "high": 128 },
"4096": { "low": 1, "high": 64 },
"16384": { "low": 1, "high": 32 },
"65536": { "low": 1, "high": 8 },
"131072": { "low": 1, "high": 4 },
"262144": { "low": 1, "high": 2 },
"524288": { "low": 1, "high": 2 },
"1048576": { "low": 1, "high": 2 }
"1024": {
"low": 1,
"high": 128
},
"4096": {
"low": 1,
"high": 64
},
"8192": {
"low": 1,
"high": 64
},
"16384": {
"low": 1,
"high": 32
},
"32768": {
"low": 1,
"high": 16
},
"65536": {
"low": 1,
"high": 8
},
"131072": {
"low": 1,
"high": 4
},
"262144": {
"low": 1,
"high": 2
},
"524288": {
"low": 1,
"high": 2
},
"1048576": {
"low": 1,
"high": 2
}
}
}

67
experiments/h200/dsv4_h200_sglang_tp_dp_matrix/matrix.json
View File

@ -1,7 +1,6 @@
{
"comment": "ISL/DSL matrix for dsv4_h200_sglang_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.",
"mode": "Y",
"comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.",
"mode": "Y",
"matrix": {
"1024": {
"128": "Y",
@ -19,6 +18,14 @@
"2048": "Y",
"4096": "Y"
},
"8192": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "Y"
},
"16384": {
"128": "Y",
"256": "Y",
@ -27,6 +34,14 @@
"2048": "Y",
"4096": "P"
},
"32768": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "P"
},
"65536": {
"128": "Y",
"256": "Y",
@ -69,13 +84,45 @@
}
},
"concurrency": {
"1024": { "low": 1, "high": 128 },
"4096": { "low": 1, "high": 64 },
"16384": { "low": 1, "high": 32 },
"65536": { "low": 1, "high": 8 },
"131072": { "low": 1, "high": 4 },
"262144": { "low": 1, "high": 2 },
"524288": { "low": 1, "high": 2 },
"1048576": { "low": 1, "high": 2 }
"1024": {
"low": 1,
"high": 128
},
"4096": {
"low": 1,
"high": 64
},
"8192": {
"low": 1,
"high": 64
},
"16384": {
"low": 1,
"high": 32
},
"32768": {
"low": 1,
"high": 16
},
"65536": {
"low": 1,
"high": 8
},
"131072": {
"low": 1,
"high": 4
},
"262144": {
"low": 1,
"high": 2
},
"524288": {
"low": 1,
"high": 2
},
"1048576": {
"low": 1,
"high": 2
}
}
}

67
experiments/h200/dsv4_h200_vllm_tp_dp_matrix/matrix.json
View File

@ -1,7 +1,6 @@
{
"comment": "ISL/DSL matrix for dsv4_h200_vllm_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.",
"mode": "Y",
"comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.",
"mode": "Y",
"matrix": {
"1024": {
"128": "Y",
@ -19,6 +18,14 @@
"2048": "Y",
"4096": "Y"
},
"8192": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "Y"
},
"16384": {
"128": "Y",
"256": "Y",
@ -27,6 +34,14 @@
"2048": "Y",
"4096": "P"
},
"32768": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "P"
},
"65536": {
"128": "Y",
"256": "Y",
@ -69,13 +84,45 @@
}
},
"concurrency": {
"1024": { "low": 1, "high": 128 },
"4096": { "low": 1, "high": 64 },
"16384": { "low": 1, "high": 32 },
"65536": { "low": 1, "high": 8 },
"131072": { "low": 1, "high": 4 },
"262144": { "low": 1, "high": 2 },
"524288": { "low": 1, "high": 2 },
"1048576": { "low": 1, "high": 2 }
"1024": {
"low": 1,
"high": 128
},
"4096": {
"low": 1,
"high": 64
},
"8192": {
"low": 1,
"high": 64
},
"16384": {
"low": 1,
"high": 32
},
"32768": {
"low": 1,
"high": 16
},
"65536": {
"low": 1,
"high": 8
},
"131072": {
"low": 1,
"high": 4
},
"262144": {
"low": 1,
"high": 2
},
"524288": {
"low": 1,
"high": 2
},
"1048576": {
"low": 1,
"high": 2
}
}
}

49
experiments/p800/dsv4_p800_sglang_tp_dp_matrix/matrix.json
View File

@ -19,6 +19,14 @@
"2048": "Y",
"4096": "Y"
},
"8192": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "Y"
},
"16384": {
"128": "Y",
"256": "Y",
@ -27,6 +35,14 @@
"2048": "Y",
"4096": "P"
},
"32768": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "P"
},
"65536": {
"128": "Y",
"256": "Y",
@ -45,10 +61,33 @@
}
},
"concurrency": {
"1024": { "low": 1, "high": 128 },
"4096": { "low": 1, "high": 64 },
"16384": { "low": 1, "high": 32 },
"65536": { "low": 1, "high": 8 },
"131072": { "low": 1, "high": 4 }
"1024": {
"low": 1,
"high": 128
},
"4096": {
"low": 1,
"high": 64
},
"8192": {
"low": 1,
"high": 64
},
"16384": {
"low": 1,
"high": 32
},
"32768": {
"low": 1,
"high": 16
},
"65536": {
"low": 1,
"high": 8
},
"131072": {
"low": 1,
"high": 4
}
}
}

7
experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env
View File

@ -15,11 +15,16 @@ NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"
TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}"
PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}"
# Stop a shape when p95 TTFT exceeds the SLO; keep group skipping disabled.
TTFT_SLO_MS="${TTFT_SLO_MS:-4000}"
ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}"
TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-0}"
# Keep the same random workload semantics as the fixed matrix baseline.
# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times
# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to
# random-ids to use generated token IDs without a ShareGPT seed dataset.
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random-ids}"
# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed
# ISL/OSL points; lower values intentionally benchmark a length distribution.
RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}"

2
experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/compare.py
View File

@ -83,7 +83,7 @@ def main():
f.write(f"- Model: `{model}`\n")
f.write(f"- Hardware: {hardware}\n")
f.write("- Backend: SGLang (Docker)\n")
f.write("- Benchmark client: `sglang.bench_serving`\n")
f.write("- Benchmark client: `sglang.benchmark.serving`\n")
f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n")
# Configuration overview.

16
experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env
View File

@ -7,7 +7,7 @@
EXPERIMENT="dsv4_pro6000_sglang_tp_dp_matrix"
MODEL_NAME="DeepSeek-V4-Flash"
MODEL_PATH="/root/data/models/ds_flash_v4"
MODEL_PATH="/data/6000D/DeepSeek-V4-Flash"
SERVED_MODEL_NAME="deepseek-v4-flash"
SGLANG_PORT="${SGLANG_PORT:-30031}"
@ -20,7 +20,7 @@ VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}"
# Run the benchmark client natively (0) or inside Docker (1).
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,3,4,5,6,7}"
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
# Runtime working directory for logs, pid files, and tmp. Defaults to a local
# directory under this experiment so the benchmark is self-contained.
@ -41,12 +41,14 @@ MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}"
# Deployment switch. 0 = native sglang venv, 1 = Docker.
USE_DOCKER="${USE_DOCKER:-1}"
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}"
DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
# Dataset used by sglang.bench_serving --dataset-name random.
# The random sampler needs a ShareGPT-style JSON file locally; it falls back to
# downloading from HuggingFace, which usually fails on offline nodes.
DATASET_PATH="${DATASET_PATH:-/root/data/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}"
# Offline by default: random-ids generates token IDs without downloading data.
# To use ShareGPT, set BENCH_DATASET_NAME=random and DATASET_PATH explicitly.
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random-ids}"
DATASET_PATH="${DATASET_PATH:-}"
SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}"
CACHE_DIR="${CACHE_DIR:-/data/yy/sskj/sglang_sm120_cache}"
# Matrix and concurrency rules are defined in matrix.json by default.
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"

67
experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/matrix.json
View File

@ -1,7 +1,6 @@
{
"comment": "ISL/DSL matrix for dsv4_h200_sglang_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.",
"mode": "Y",
"comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.",
"mode": "Y",
"matrix": {
"1024": {
"128": "Y",
@ -19,6 +18,14 @@
"2048": "Y",
"4096": "Y"
},
"8192": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "Y"
},
"16384": {
"128": "Y",
"256": "Y",
@ -27,6 +34,14 @@
"2048": "Y",
"4096": "P"
},
"32768": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "P"
},
"65536": {
"128": "Y",
"256": "Y",
@ -69,13 +84,45 @@
}
},
"concurrency": {
"1024": { "low": 1, "high": 128 },
"4096": { "low": 1, "high": 64 },
"16384": { "low": 1, "high": 32 },
"65536": { "low": 1, "high": 8 },
"131072": { "low": 1, "high": 4 },
"262144": { "low": 1, "high": 2 },
"524288": { "low": 1, "high": 2 },
"1048576": { "low": 1, "high": 2 }
"1024": {
"low": 1,
"high": 128
},
"4096": {
"low": 1,
"high": 64
},
"8192": {
"low": 1,
"high": 64
},
"16384": {
"low": 1,
"high": 32
},
"32768": {
"low": 1,
"high": 16
},
"65536": {
"low": 1,
"high": 8
},
"131072": {
"low": 1,
"high": 4
},
"262144": {
"low": 1,
"high": 2
},
"524288": {
"low": 1,
"high": 2
},
"1048576": {
"low": 1,
"high": 2
}
}
}

11
experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh
View File

@ -27,7 +27,7 @@ else
PYTHON="$(command -v python3)"
fi
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}"
DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
engine_is_healthy() {
curl --fail --silent --show-error --max-time 5 \
@ -68,7 +68,7 @@ engine_build_server_args() {
local tp="$1"
local dp="$2"
local -a args=(
sglang serve --model-path "$MODEL_PATH"
python3 -m sglang.launch_server --model-path "$MODEL_PATH"
--trust-remote-code
--tp-size "$tp"
--moe-runner-backend "$MOE_RUNNER_BACKEND"
@ -167,15 +167,16 @@ engine_run_bench() {
--network host \
"${volume_args[@]}" \
-e PYTHONUNBUFFERED=1 \
--entrypoint python3 \
"$DOCKER_IMAGE" \
python -m sglang.bench_serving "${bench_args[@]}"
-m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
else
"$PYTHON" -m sglang.bench_serving "${bench_args[@]}"
"$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
fi
}
export -f engine_run_bench
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
adaptive_main "$@"

11
experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh
View File

@ -27,7 +27,7 @@ else
PYTHON="$(command -v python3)"
fi
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}"
DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
engine_is_healthy() {
curl --fail --silent --show-error --max-time 5 \
@ -68,7 +68,7 @@ engine_build_server_args() {
local tp="$1"
local dp="$2"
local -a args=(
sglang serve --model-path "$MODEL_PATH"
python3 -m sglang.launch_server --model-path "$MODEL_PATH"
--trust-remote-code
--tp-size "$tp"
--moe-runner-backend "$MOE_RUNNER_BACKEND"
@ -167,16 +167,17 @@ engine_run_bench() {
--network host \
"${volume_args[@]}" \
-e PYTHONUNBUFFERED=1 \
--entrypoint python3 \
"$DOCKER_IMAGE" \
python -m sglang.bench_serving "${bench_args[@]}"
-m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
else
"$PYTHON" -m sglang.bench_serving "${bench_args[@]}"
"$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
fi
}
export -f engine_run_bench
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
export SEARCH_START_CONCURRENCY=16
export SEARCH_ADDEND=16

41
experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh
View File

@ -22,7 +22,7 @@ DRY_RUN="${DRY_RUN:-0}"
GRID_LIMIT="${GRID_LIMIT:-0}"
# Export variables used inside functions that are called via bash -c subshells.
export DATASET_PATH MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT
export BENCH_DATASET_NAME DATASET_PATH MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT SGLANG_BENCH_MODULE
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
PYTHON="${VENV_CLIENT}/bin/python"
@ -30,7 +30,7 @@ else
PYTHON="$(command -v python3)"
fi
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}"
DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
log_dir_global="${RESULT_BASE}/${RUN_ID}/logs"
mkdir -p "$log_dir_global"
@ -79,9 +79,9 @@ stop_server() {
docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
fi
# Fallback: kill any sglang serve processes for this model.
pkill -9 -f "sglang serve.*${MODEL_NAME}" 2>/dev/null || true
pkill -9 -f "sglang serve.*${MODEL_PATH}" 2>/dev/null || true
# Fallback: kill any SGLang launch processes for this model.
pkill -9 -f "sglang.launch_server.*${MODEL_NAME}" 2>/dev/null || true
pkill -9 -f "sglang.launch_server.*${MODEL_PATH}" 2>/dev/null || true
sleep 2
}
@ -90,7 +90,7 @@ build_server_args() {
local dp="$2"
local args=(
"sglang serve" "$MODEL_PATH"
"python3 -m sglang.launch_server" --model-path "$MODEL_PATH"
--trust-remote-code
--tp-size "$tp"
--moe-runner-backend "$MOE_RUNNER_BACKEND"
@ -138,11 +138,17 @@ restart_server() {
}
run_bench_serving() {
# Run sglang.bench_serving either natively or inside the SGLang Docker image.
# Inject the offline workload choice consistently for warmup and main runs.
local -a dataset_args=(--dataset-name "$BENCH_DATASET_NAME")
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
dataset_args+=(--dataset-path "$DATASET_PATH")
else
dataset_args+=(--tokenize-prompt)
fi
if [[ "${USE_DOCKER_CLIENT:-1}" == "1" ]]; then
local vol_args=()
vol_args+=("-v" "${MODEL_PATH}:${MODEL_PATH}:ro")
if [[ -n "${DATASET_PATH:-}" ]]; then
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
vol_args+=("-v" "${DATASET_PATH}:${DATASET_PATH}:ro")
fi
vol_args+=("-v" "${RESULT_BASE}:${RESULT_BASE}")
@ -150,10 +156,11 @@ run_bench_serving() {
--network host \
"${vol_args[@]}" \
-e PYTHONUNBUFFERED=1 \
--entrypoint python3 \
"${DOCKER_IMAGE}" \
python -m sglang.bench_serving "$@"
-m "$SGLANG_BENCH_MODULE" "${dataset_args[@]}" "$@"
else
"$PYTHON" -m sglang.bench_serving "$@"
"$PYTHON" -m "$SGLANG_BENCH_MODULE" "${dataset_args[@]}" "$@"
fi
}
export -f run_bench_serving
@ -168,8 +175,6 @@ run_warmup() {
--backend sglang \
--host 127.0.0.1 \
--port "'"$SGLANG_PORT"'" \
--dataset-name random \
--dataset-path "'"$DATASET_PATH"'" \
--random-input-len "'"$input_len"'" \
--random-output-len "'"$output_len"'" \
--num-prompts 1 \
@ -319,7 +324,7 @@ skip_remaining_scenarios() {
fi
append_scenario_record "$result_root" \
"name=${sname}" \
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
"status=\"${status}\"" \
"note=\"${reason}\""
record_skipped_csv "$skipped_csv" \
@ -447,8 +452,6 @@ run_parallel_config() {
--backend sglang \
--host 127.0.0.1 \
--port "'"$SGLANG_PORT"'" \
--dataset-name random \
--dataset-path "'"$DATASET_PATH"'" \
--random-input-len "'"$isl"'" \
--random-output-len "'"$dsl"'" \
--num-prompts "'"$num"'" \
@ -465,7 +468,7 @@ run_parallel_config() {
log "finished ${config_label} scenario: output=${output_file}"
append_scenario_record "$result_root" \
"name=${sname}" \
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
"status=\"completed\"" \
"note=\"benchmark finished successfully\""
continue
@ -476,7 +479,7 @@ run_parallel_config() {
log "ERROR: ${config_label} scenario ${sname} triggered OOM; stopping config"
append_scenario_record "$result_root" \
"name=${sname}" \
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
"status=\"OOM\"" \
"note=\"detected CUDA out-of-memory\""
record_skipped_csv "$skipped_csv" \
@ -491,7 +494,7 @@ run_parallel_config() {
log "optional (P) scenario failed; recording as skipped and continuing"
append_scenario_record "$result_root" \
"name=${sname}" \
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
"status=\"skipped_optional\"" \
"note=\"optional scenario failed (rc=${bench_rc})\""
record_skipped_csv "$skipped_csv" \
@ -509,7 +512,7 @@ run_parallel_config() {
log "ERROR: ${config_label} server restart failed; skipping remaining scenarios"
append_scenario_record "$result_root" \
"name=${sname}" \
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
"status=\"FAILED\"" \
"note=\"scenario failed and server restart failed (rc=${bench_rc})\""
record_skipped_csv "$skipped_csv" \

12
experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh
View File

@ -22,7 +22,7 @@ rm -f "$PID_FILE"
# Build server args.
SERVER_ARGS=(
serve
-m sglang.launch_server
--model-path "$MODEL_PATH"
--trust-remote-code
--tp-size "$TP"
@ -44,24 +44,24 @@ SERVER_ARGS_STR="${SERVER_ARGS[*]}"
echo "=== Starting SGLang server inside container (TP=${TP}, DP=${DP}) ==="
echo "Container: $NAME"
echo "Command: sglang ${SERVER_ARGS_STR}"
echo "Command: python3 ${SERVER_ARGS_STR}"
echo "Log: $LOG"
# Kill any existing sglang process inside container first.
docker exec "$NAME" pkill -9 -f "sglang serve" 2>/dev/null || true
docker exec "$NAME" pkill -9 -f "sglang.launch_server" 2>/dev/null || true
sleep 3
# Start sglang serve inside the container in background.
# Start the SGLang launch module inside the container in background.
# We use nohup so it survives after docker exec returns.
docker exec -d \
-e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
-e PYTHONUNBUFFERED=1 \
"$NAME" \
bash -c "nohup sglang ${SERVER_ARGS_STR} > /tmp/sglang_server.log 2>&1 &"
bash -c "nohup python3 ${SERVER_ARGS_STR} > /tmp/sglang_server.log 2>&1 &"
# Wait for the server process to appear inside container.
sleep 2
SERVER_PID=$(docker exec "$NAME" pgrep -f "sglang serve" | head -n 1 || true)
SERVER_PID=$(docker exec "$NAME" pgrep -f "sglang.launch_server" | head -n 1 || true)
if [[ -z "$SERVER_PID" ]]; then
echo "ERROR: sglang server process not found inside container"
docker exec "$NAME" cat /tmp/sglang_server.log 2>/dev/null | tail -50 || true

20
experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_container.sh
View File

@ -13,14 +13,13 @@ source "${SCRIPT_DIR}/config.env"
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}"
IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
PORT="${SGLANG_PORT:-30031}"
NAME="${EXPERIMENT}_sglang_container"
PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_container.pid"
# Persistent cache directory on host for DeepGEMM JIT kernels.
CACHE_DIR="${RUNTIME_BASE}/cache"
mkdir -p "${CACHE_DIR}/deep_gemm" "${CACHE_DIR}/tvm-ffi"
# Persistent cache directory for SM120 JIT kernels and tokenizer metadata.
mkdir -p "$CACHE_DIR"
# Clean up any stale container.
docker rm -f "$NAME" >/dev/null 2>&1 || true
@ -40,16 +39,19 @@ echo "Log: $LOG"
docker run -d \
--name "$NAME" \
--gpus all \
--ipc host \
--shm-size 16g \
--privileged \
--ipc=host \
--network host \
--ulimit memlock=-1 \
--ulimit stack=67108864 \
--entrypoint /bin/bash \
-p "${PORT}:${PORT}" \
-v "${MODEL_PATH}:${MODEL_PATH}:ro" \
-v "${CACHE_DIR}/deep_gemm:/root/.cache/deep_gemm" \
-v "${CACHE_DIR}/tvm-ffi:/root/.cache/tvm-ffi" \
-v "${CACHE_DIR}:/root/.cache" \
-v "${RUNTIME_BASE}/tmp:/tmp" \
-e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
-e PYTHONUNBUFFERED=1 \
-e HF_HUB_OFFLINE=1 \
-e TRANSFORMERS_OFFLINE=1 \
"$IMAGE" \
-c "sleep infinity" \
> "$LOG" 2>&1

25
experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh
View File

@ -2,8 +2,8 @@
# Start SGLang server in Docker for a given TPxDP configuration.
# Usage: start_sglang_docker.sh <TP> <DP>
#
# Uses the lmsysorg/sglang image and the same argument set as the
# bare-metal start script. The container is removed automatically on stop.
# Uses the verified SGLang SM120 image and keeps its JIT cache on
# persistent storage. The container is removed automatically on stop.
set -e
TP="${1}"
@ -14,9 +14,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${SCRIPT_DIR}/config.env"
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" "$CACHE_DIR"
IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:latest}"
IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
PORT="${SGLANG_PORT:-30031}"
NAME="${EXPERIMENT}_sglang_tp${TP}_dp${DP}"
PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid"
@ -28,7 +28,7 @@ rm -f "$PID_FILE"
docker rm -f "$NAME" >/dev/null 2>&1 || true
SERVER_ARGS=(
serve
-m sglang.launch_server
--model-path "$MODEL_PATH"
--trust-remote-code
--tp-size "$TP"
@ -53,7 +53,7 @@ echo "Image: $IMAGE"
echo "Model: $MODEL_PATH"
echo "Container name: $NAME"
echo "Host port: $PORT"
echo "Command: sglang ${SERVER_ARGS_STR}"
echo "Command: python3 ${SERVER_ARGS_STR}"
echo "Log: $LOG"
# Run docker in the foreground so that killing the host process stops the
@ -62,14 +62,19 @@ echo "Log: $LOG"
nohup docker run --rm \
--name "$NAME" \
--gpus all \
--ipc host \
--shm-size 16g \
--entrypoint sglang \
-p "${PORT}:${PORT}" \
--privileged \
--ipc=host \
--network host \
--ulimit memlock=-1 \
--ulimit stack=67108864 \
--entrypoint python3 \
-v "${MODEL_PATH}:${MODEL_PATH}:ro" \
-v "${CACHE_DIR}:/root/.cache" \
-v "${RUNTIME_BASE}/tmp:/tmp" \
-e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
-e PYTHONUNBUFFERED=1 \
-e HF_HUB_OFFLINE=1 \
-e TRANSFORMERS_OFFLINE=1 \
"$IMAGE" \
"${SERVER_ARGS[@]}" \
> "$LOG" 2>&1 &

2
experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh
View File

@ -33,7 +33,7 @@ PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid"
rm -f "$PID_FILE"
SERVER_ARGS=(
sglang serve
python3 -m sglang.launch_server
--model-path "$MODEL_PATH"
--trust-remote-code
--tp-size "$TP"

7
experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env
View File

@ -15,11 +15,16 @@ NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"
TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}"
PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}"
# Stop a shape when p95 TTFT exceeds the SLO; keep group skipping disabled.
TTFT_SLO_MS="${TTFT_SLO_MS:-4000}"
ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}"
TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-0}"
# Keep the same random workload semantics as the fixed matrix baseline.
# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times
# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to
# random-ids to use generated token IDs without a ShareGPT seed dataset.
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random-ids}"
# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed
# ISL/OSL points; lower values intentionally benchmark a length distribution.
RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}"

18
experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/config.env
View File

@ -6,7 +6,7 @@
EXPERIMENT="dsv4_pro6000_vllm_tp_dp_matrix"
MODEL_NAME="DeepSeek-V4-Flash"
MODEL_PATH="/data/hf_models/DeepSeek-V4-Flash"
MODEL_PATH="/data/6000D/DeepSeek-V4-Flash"
SERVED_MODEL_NAME="deepseek-v4-flash"
VLLM_PORT="${VLLM_PORT:-30030}"
@ -41,17 +41,19 @@ BLOCK_SIZE="${BLOCK_SIZE:-256}"
# Deployment switch. 0 = native vllm venv, 1 = Docker.
USE_DOCKER="${USE_DOCKER:-1}"
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}"
# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving,
# Benchmark client Docker image. vLLM's image does not include the SGLang benchmark client,
# so the client runs inside the SGLang image and targets the vLLM backend via the
# OpenAI-compatible API.
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
# Dataset used by sglang.bench_serving --dataset-name random.
# The random sampler needs a ShareGPT-style JSON file locally; it falls back to
# downloading from HuggingFace, which usually fails on offline nodes.
DATASET_PATH="${DATASET_PATH:-/data/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}"
# Offline by default: random-ids generates token IDs without downloading data.
# To use ShareGPT, set BENCH_DATASET_NAME=random and DATASET_PATH explicitly.
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random-ids}"
DATASET_PATH="${DATASET_PATH:-}"
SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}"
CACHE_DIR="${CACHE_DIR:-/data/yy/sskj/vllm_sm120_cache}"
# Matrix and concurrency rules are defined in matrix.json by default.
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"

128
experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/matrix.json Normal file
View File

@ -0,0 +1,128 @@
{
"comment": "ISL/OSL matrix for dsv4_pro6000_vllm_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.",
"mode": "Y",
"matrix": {
"1024": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "Y"
},
"4096": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "Y"
},
"8192": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "Y"
},
"16384": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "P"
},
"32768": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "P"
},
"65536": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "P",
"4096": "N"
},
"131072": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "P",
"2048": "N",
"4096": "N"
},
"262144": {
"128": "Y",
"256": "Y",
"512": "P",
"1024": "N",
"2048": "N",
"4096": "N"
},
"524288": {
"128": "Y",
"256": "P",
"512": "N",
"1024": "N",
"2048": "N",
"4096": "N"
},
"1048576": {
"128": "Y",
"256": "P",
"512": "N",
"1024": "N",
"2048": "N",
"4096": "N"
}
},
"concurrency": {
"1024": {
"low": 1,
"high": 128
},
"4096": {
"low": 1,
"high": 64
},
"8192": {
"low": 1,
"high": 64
},
"16384": {
"low": 1,
"high": 32
},
"32768": {
"low": 1,
"high": 16
},
"65536": {
"low": 1,
"high": 8
},
"131072": {
"low": 1,
"high": 4
},
"262144": {
"low": 1,
"high": 2
},
"524288": {
"low": 1,
"high": 2
},
"1048576": {
"low": 1,
"high": 2
}
}
}

12
experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh
View File

@ -27,8 +27,8 @@ else
PYTHON="$(command -v python3)"
fi
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}"
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
engine_is_healthy() {
curl --fail --silent --show-error --max-time 5 \
@ -68,7 +68,6 @@ engine_build_server_args() {
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
--max-model-len "$MAX_MODEL_LEN"
--max-num-seqs "$MAX_NUM_SEQS"
--no-enable-flashinfer-autotune
--host 0.0.0.0
--port "$ENGINE_PORT"
)
@ -150,15 +149,16 @@ engine_run_bench() {
-e PYTHONUNBUFFERED=1 \
-e HF_HUB_OFFLINE=1 \
-e TRANSFORMERS_OFFLINE=1 \
--entrypoint python3 \
"$DOCKER_CLIENT_IMAGE" \
python -m sglang.bench_serving "${bench_args[@]}"
-m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
else
"$PYTHON" -m sglang.bench_serving "${bench_args[@]}"
"$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
fi
}
export -f engine_run_bench
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
adaptive_main "$@"

12
experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh
View File

@ -27,8 +27,8 @@ else
PYTHON="$(command -v python3)"
fi
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}"
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
engine_is_healthy() {
curl --fail --silent --show-error --max-time 5 \
@ -68,7 +68,6 @@ engine_build_server_args() {
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
--max-model-len "$MAX_MODEL_LEN"
--max-num-seqs "$MAX_NUM_SEQS"
--no-enable-flashinfer-autotune
--host 0.0.0.0
--port "$ENGINE_PORT"
)
@ -150,16 +149,17 @@ engine_run_bench() {
-e PYTHONUNBUFFERED=1 \
-e HF_HUB_OFFLINE=1 \
-e TRANSFORMERS_OFFLINE=1 \
--entrypoint python3 \
"$DOCKER_CLIENT_IMAGE" \
python -m sglang.bench_serving "${bench_args[@]}"
-m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
else
"$PYTHON" -m sglang.bench_serving "${bench_args[@]}"
"$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
fi
}
export -f engine_run_bench
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
export SEARCH_START_CONCURRENCY=16
export SEARCH_ADDEND=16

17
experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh
View File

@ -2,8 +2,8 @@
# Start vLLM server in Docker for a given TPxDP configuration.
# Usage: start_vllm_docker.sh <TP> <DP>
#
# Uses the vllm/vllm-openai image and the same argument set as the
# bare-metal start script. The container is removed automatically on stop.
# Uses the verified vLLM SM120 image and keeps its cache on persistent
# storage. The container is removed automatically on stop.
set -e
TP="${1}"
@ -14,9 +14,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${SCRIPT_DIR}/config.env"
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" "$CACHE_DIR"
IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}"
PORT="${VLLM_PORT:-30030}"
NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}"
PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"
@ -36,7 +36,6 @@ SERVER_ARGS=(
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
--max-model-len "$MAX_MODEL_LEN"
--max-num-seqs "$MAX_NUM_SEQS"
--no-enable-flashinfer-autotune
--host 0.0.0.0
--port "$PORT"
)
@ -63,12 +62,14 @@ echo "Log: $LOG"
nohup docker run --rm \
--name "$NAME" \
--gpus all \
--ipc host \
--shm-size 16g \
--privileged \
--ipc=host \
--network host \
--ulimit memlock=-1 \
--ulimit stack=67108864 \
--entrypoint vllm \
-p "${PORT}:${PORT}" \
-v "${MODEL_PATH}:${MODEL_PATH}:ro" \
-v "${CACHE_DIR}:/root/.cache" \
-v "${RUNTIME_BASE}/tmp:/tmp" \
-e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
-e PYTHONUNBUFFERED=1 \

1
experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh
View File

@ -41,7 +41,6 @@ SERVER_ARGS=(
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
--max-model-len "$MAX_MODEL_LEN"
--max-num-seqs "$MAX_NUM_SEQS"
--no-enable-flashinfer-autotune
--host 0.0.0.0
--port "$VLLM_PORT"
)

4
scripts/common/platform.sh
View File

@ -21,7 +21,9 @@ if [[ -z "${PLATFORM:-}" ]]; then
PLATFORM="kunlun_p800"
elif command -v nvidia-smi >/dev/null 2>&1; then
_GPU_NAME=$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -n 1 || true)
if [[ -n "$_GPU_NAME" ]] && grep -qi "H20" <<< "$_GPU_NAME"; then
if [[ -n "$_GPU_NAME" ]] && grep -qiE "RTX 6000D|PRO 6000D" <<< "$_GPU_NAME"; then
PLATFORM="nvidia_rtx6000d"
elif [[ -n "$_GPU_NAME" ]] && grep -qi "H20" <<< "$_GPU_NAME"; then
PLATFORM="nvidia_h20"
else
PLATFORM="nvidia_h200"