dsv4_h200_vllm_tp_dp_matrix: use vLLM internal DP LB, drop proxy/supervisor
This commit is contained in:
yy-fighting
parent
59dbadb110
commit
329d4c3975
@ -10,11 +10,6 @@ MODEL_PATH="/data/models/DeepSeek-V4-Flash"
|
||||
SERVED_MODEL_NAME="deepseek-v4-flash"
|
||||
|
||||
VLLM_PORT="${VLLM_PORT:-30030}"
|
||||
# vLLM 0.24.0 multi-port DP supervisor listens on this hard-coded port.
|
||||
VLLM_DP_SUPERVISOR_PORT="${VLLM_DP_SUPERVISOR_PORT:-9256}"
|
||||
# The bench client does NOT talk to the supervisor directly; we put a
|
||||
# round-robin proxy in front of the DP child servers.
|
||||
VLLM_DP_PROXY_PORT="${VLLM_DP_PROXY_PORT:-30040}"
|
||||
|
||||
VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}"
|
||||
VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}"
|
||||
|
||||
@ -32,44 +32,14 @@ log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE}"
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Port used by run_bench to check the vLLM server (supervisor for DP>1).
|
||||
server_health_port() {
|
||||
local dp="$1"
|
||||
if [[ "$dp" -gt 1 ]]; then
|
||||
echo "$VLLM_DP_SUPERVISOR_PORT"
|
||||
else
|
||||
echo "$VLLM_PORT"
|
||||
fi
|
||||
}
|
||||
|
||||
# Port used by the benchmark client (proxy for DP>1).
|
||||
bench_port() {
|
||||
local dp="$1"
|
||||
if [[ "$dp" -gt 1 ]]; then
|
||||
echo "$VLLM_DP_PROXY_PORT"
|
||||
else
|
||||
echo "$VLLM_PORT"
|
||||
fi
|
||||
}
|
||||
|
||||
is_server_healthy() {
|
||||
local port="$1"
|
||||
curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
is_proxy_healthy() {
|
||||
local port="$1"
|
||||
# The bench client uses /v1/models as the ready signal, so the proxy must
|
||||
# forward it successfully.
|
||||
curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/v1/models" >/dev/null 2>&1
|
||||
curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
stop_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
|
||||
stop_proxy "$tp" "$dp"
|
||||
|
||||
local pid_file="/data/user1/yy/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
|
||||
if [[ -f "$pid_file" ]]; then
|
||||
local pid
|
||||
@ -89,74 +59,6 @@ stop_server() {
|
||||
sleep 2
|
||||
}
|
||||
|
||||
start_proxy() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
if [[ "$dp" -le 1 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local proxy_port="$VLLM_DP_PROXY_PORT"
|
||||
local base_port="$VLLM_PORT"
|
||||
local backends=""
|
||||
for ((r = 0; r < dp; r++)); do
|
||||
backends="${backends:+,}$((base_port + r))"
|
||||
done
|
||||
|
||||
local pid_file="/data/user1/yy/${EXPERIMENT}_proxy_tp${tp}_dp${dp}.pid"
|
||||
rm -f "$pid_file"
|
||||
|
||||
log "starting DP proxy on port ${proxy_port} -> backends ${backends}"
|
||||
nohup "${VENV_VLLM}/bin/python" "${SCRIPT_DIR}/../../scripts/common/dp_proxy.py" \
|
||||
--port "$proxy_port" \
|
||||
--backends "$backends" \
|
||||
--timeout 300 \
|
||||
> "${log_dir_global}/vllm_tp${tp}_dp${dp}.proxy.log" 2>&1 &
|
||||
|
||||
local pid=$!
|
||||
echo $pid > "$pid_file"
|
||||
|
||||
for i in $(seq 1 60); do
|
||||
if is_proxy_healthy "$proxy_port"; then
|
||||
log "DP proxy is ready on port ${proxy_port}"
|
||||
return 0
|
||||
fi
|
||||
if ! kill -0 "$pid" 2>/dev/null; then
|
||||
log "ERROR: DP proxy exited early"
|
||||
tail -100 "${log_dir_global}/vllm_tp${tp}_dp${dp}.proxy.log"
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
log "ERROR: DP proxy not healthy after 60s"
|
||||
tail -100 "${log_dir_global}/vllm_tp${tp}_dp${dp}.proxy.log"
|
||||
return 1
|
||||
}
|
||||
|
||||
stop_proxy() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
if [[ "$dp" -le 1 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local pid_file="/data/user1/yy/${EXPERIMENT}_proxy_tp${tp}_dp${dp}.pid"
|
||||
if [[ -f "$pid_file" ]]; then
|
||||
local pid
|
||||
pid="$(cat "$pid_file")"
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
log "stopping DP proxy pid=${pid}"
|
||||
kill "$pid" 2>/dev/null || true
|
||||
sleep 2
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "$pid_file"
|
||||
fi
|
||||
pkill -9 -f "dp_proxy.py.*--port ${VLLM_DP_PROXY_PORT}" 2>/dev/null || true
|
||||
sleep 1
|
||||
}
|
||||
|
||||
build_server_args() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
@ -173,8 +75,6 @@ build_server_args() {
|
||||
if [[ "$dp" -gt 1 ]]; then
|
||||
args+=(
|
||||
--data-parallel-size "$dp"
|
||||
--data-parallel-size-local "$dp"
|
||||
--data-parallel-multi-port-external-lb
|
||||
)
|
||||
fi
|
||||
printf '%s ' "${args[@]}"
|
||||
@ -188,20 +88,11 @@ start_server() {
|
||||
bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" \
|
||||
>> "${log_dir_global}/vllm_tp${tp}_dp${dp}.server.outer.log" 2>&1
|
||||
|
||||
local hport
|
||||
hport="$(server_health_port "$dp")"
|
||||
if ! is_server_healthy "$hport"; then
|
||||
log "error: vllm server tp=${tp} dp=${dp} failed health check on port ${hport}"
|
||||
if ! is_server_healthy; then
|
||||
log "error: vllm server tp=${tp} dp=${dp} failed health check on port ${VLLM_PORT}"
|
||||
return 1
|
||||
fi
|
||||
log "vllm server tp=${tp} dp=${dp} is healthy on port ${hport}"
|
||||
|
||||
if [[ "$dp" -gt 1 ]]; then
|
||||
if ! start_proxy "$tp" "$dp"; then
|
||||
log "error: failed to start DP proxy for tp=${tp} dp=${dp}"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
log "vllm server tp=${tp} dp=${dp} is healthy on port ${VLLM_PORT}"
|
||||
}
|
||||
|
||||
run_warmup() {
|
||||
@ -209,14 +100,12 @@ run_warmup() {
|
||||
local dp="$2"
|
||||
local input_len="$3"
|
||||
local output_len="$4"
|
||||
local bport
|
||||
bport="$(bench_port "$dp")"
|
||||
|
||||
log "warming up tp=${tp} dp=${dp} (input=${input_len}, output=${output_len}, num=1)"
|
||||
"$PYTHON" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \
|
||||
--backend vllm \
|
||||
--host 127.0.0.1 \
|
||||
--port "$bport" \
|
||||
--port "$VLLM_PORT" \
|
||||
--input-len "$input_len" \
|
||||
--output-len "$output_len" \
|
||||
--num 1 \
|
||||
@ -393,7 +282,7 @@ run_parallel_config() {
|
||||
"$PYTHON" -m sglang.bench_serving \
|
||||
--backend vllm \
|
||||
--host 127.0.0.1 \
|
||||
--port "$(bench_port "$dp")" \
|
||||
--port "$VLLM_PORT" \
|
||||
--dataset-name random \
|
||||
--random-input-len "$isl" \
|
||||
--random-output-len "$dsl" \
|
||||
|
||||
@ -8,7 +8,8 @@
|
||||
# --block-size 256
|
||||
# --tensor-parallel-size <TP>
|
||||
# --no-enable-flashinfer-autotune
|
||||
# plus DP-related flags when DP > 1.
|
||||
# When DP > 1, only --data-parallel-size is added so vLLM uses its internal
|
||||
# DP load balancer (single API server, single port).
|
||||
set -e
|
||||
|
||||
TP="${1}"
|
||||
@ -45,20 +46,14 @@ SERVER_ARGS=(
|
||||
if [[ "$DP" -gt 1 ]]; then
|
||||
SERVER_ARGS+=(
|
||||
--data-parallel-size "$DP"
|
||||
--data-parallel-size-local "$DP"
|
||||
--data-parallel-multi-port-external-lb
|
||||
)
|
||||
# vLLM 0.24.0 hard-codes the DP supervisor port to 9256.
|
||||
HEALTH_PORT="$VLLM_DP_SUPERVISOR_PORT"
|
||||
else
|
||||
HEALTH_PORT="$VLLM_PORT"
|
||||
fi
|
||||
|
||||
SERVER_ARGS_STR="${SERVER_ARGS[*]}"
|
||||
|
||||
echo "=== Starting vLLM server (TP=${TP}, DP=${DP}) ==="
|
||||
echo "Model: $MODEL_PATH"
|
||||
echo "Health port: $HEALTH_PORT"
|
||||
echo "Port: $VLLM_PORT"
|
||||
echo "Command: $SERVER_ARGS_STR"
|
||||
echo "Log: $LOG"
|
||||
|
||||
@ -67,11 +62,11 @@ nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 &
|
||||
PID=$!
|
||||
echo $PID > "$PID_FILE"
|
||||
echo "PID: $PID"
|
||||
echo "Waiting for health on port ${HEALTH_PORT}..."
|
||||
echo "Waiting for health on port ${VLLM_PORT}..."
|
||||
|
||||
for i in $(seq 1 240); do
|
||||
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${HEALTH_PORT}/health" >/dev/null 2>&1; then
|
||||
echo "vLLM server is ready at http://127.0.0.1:${HEALTH_PORT}"
|
||||
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then
|
||||
echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}"
|
||||
echo "Log: $LOG"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user