[Test] Add exact Kimi SM120 PR validation point
This commit is contained in:
Zhiyi Hong
parent
d28db48e4b
commit
e8ff3ce1e8
@ -1,5 +1,7 @@
|
|||||||
# sskj — 多平台大模型推理性能基准测试项目
|
# sskj — 多平台大模型推理性能基准测试项目
|
||||||
|
|
||||||
|
**更新（2026-08-19 10:45:18 CST）**：新增 Kimi-K3 SM120 SGLang Draft PR 的单点代表性验收入口。使用精确 SGLang `300c87a`、FlashInfer #4460 实现 `b525c51` 和统一镜像，在 601-604 四节点 TP32×EP4 上仅复测 16K→1、C=8、Chunk=8K 的 Marlin/FlashInfer A/B，各 3 次重复；完整命令、镜像身份、原始日志和汇总统一落入单个 Run 目录。
|
||||||
|
|
||||||
**更新（2026-08-18 23:00:42 CST）**：完成依赖 FlashInfer #4460 的 SGLang Draft PR 收敛。确认不提交任何 FlashInfer PR，也不 vendor 或 pin 未合并 kernel；在 601 GPU6 上从 #4460 源码构建 FlashInfer 0.6.18 wheel，SGLang 定向测试 8/8 通过。Draft 标题、依赖、兼容策略、测试和四机 EP4 数据见 `experiments/pro6000/kimi3_pro6000_sglang_sm120_flashinfer_mxfp4/SGLANG_DRAFT_PR.md`。
|
**更新（2026-08-18 23:00:42 CST）**：完成依赖 FlashInfer #4460 的 SGLang Draft PR 收敛。确认不提交任何 FlashInfer PR，也不 vendor 或 pin 未合并 kernel；在 601 GPU6 上从 #4460 源码构建 FlashInfer 0.6.18 wheel，SGLang 定向测试 8/8 通过。Draft 标题、依赖、兼容策略、测试和四机 EP4 数据见 `experiments/pro6000/kimi3_pro6000_sglang_sm120_flashinfer_mxfp4/SGLANG_DRAFT_PR.md`。
|
||||||
|
|
||||||
**更新（2026-08-18 22:21:40 CST）**：完成 Kimi-K3 SM120 FlashInfer MXFP4 去重审计。确认 CUTLASS SiTU kernel 已由未合并的 FlashInfer PR #4460 实现，不再提交平行 kernel；SGLang 贡献收敛为 Kimi gate/up 与 scale 布局、SiTU 4.0/25.0 参数映射、非连续输入和 SM120 attention-residual guard，并保留 601-604 EP4 的全部服务级结果。详见 `experiments/pro6000/kimi3_pro6000_sglang_sm120_flashinfer_mxfp4/UPSTREAM_DUPLICATION_AUDIT.md`。
|
**更新（2026-08-18 22:21:40 CST）**：完成 Kimi-K3 SM120 FlashInfer MXFP4 去重审计。确认 CUTLASS SiTU kernel 已由未合并的 FlashInfer PR #4460 实现，不再提交平行 kernel；SGLang 贡献收敛为 Kimi gate/up 与 scale 布局、SiTU 4.0/25.0 参数映射、非连续输入和 SM120 attention-residual guard，并保留 601-604 EP4 的全部服务级结果。详见 `experiments/pro6000/kimi3_pro6000_sglang_sm120_flashinfer_mxfp4/UPSTREAM_DUPLICATION_AUDIT.md`。
|
||||||
|
|||||||
@ -0,0 +1,30 @@
|
|||||||
|
ARG BASE_IMAGE=lmsysorg/sglang:kimi-k3-fiv617situ-warm
|
||||||
|
FROM ${BASE_IMAGE}
|
||||||
|
|
||||||
|
ARG SGLANG_COMMIT
|
||||||
|
ARG FLASHINFER_COMMIT
|
||||||
|
|
||||||
|
ENV FLASHINFER_DISABLE_VERSION_CHECK=1 \
|
||||||
|
PYTHONPATH=/sgl-workspace/sglang/python \
|
||||||
|
PYTHONUNBUFFERED=1
|
||||||
|
|
||||||
|
COPY sglang_kernel-0.4.6.post1-cp310-abi3-manylinux2014_x86_64.whl /tmp/
|
||||||
|
COPY flashinfer_python-0.6.18-py3-none-any.whl /tmp/
|
||||||
|
RUN python3 -m pip install --no-deps --force-reinstall \
|
||||||
|
/tmp/sglang_kernel-0.4.6.post1-cp310-abi3-manylinux2014_x86_64.whl \
|
||||||
|
/tmp/flashinfer_python-0.6.18-py3-none-any.whl && \
|
||||||
|
rm -f /tmp/*.whl
|
||||||
|
|
||||||
|
# Replace the image's older Python package with the exact Draft tree. The
|
||||||
|
# matching 0.4.6.post1 sglang-kernel wheel is installed above.
|
||||||
|
RUN rm -rf /sgl-workspace/sglang/python/sglang
|
||||||
|
COPY sglang/ /sgl-workspace/sglang/python/sglang/
|
||||||
|
COPY test_mxfp4_sm120_cutlass.py /opt/pr-tests/test_mxfp4_sm120_cutlass.py
|
||||||
|
COPY source_identity.txt /opt/pr-build/source_identity.txt
|
||||||
|
|
||||||
|
RUN python3 -m compileall -q /sgl-workspace/sglang/python/sglang && \
|
||||||
|
python3 -c "import inspect; from flashinfer.fused_moe import cutlass_fused_moe; from flashinfer.fused_moe.core import ActivationType; assert hasattr(ActivationType, 'Situ'); assert 'situ_beta' in inspect.signature(cutlass_fused_moe).parameters; import sglang.srt.layers.quantization.mxfp4"
|
||||||
|
|
||||||
|
LABEL ai.meta-stone.purpose="Kimi-K3 SM120 SGLang Draft representative validation" \
|
||||||
|
ai.meta-stone.sglang.commit="${SGLANG_COMMIT}" \
|
||||||
|
ai.meta-stone.flashinfer.commit="${FLASHINFER_COMMIT}"
|
||||||
@ -0,0 +1,418 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Validate the exact SGLang Draft with one four-node Kimi-K3 serving point.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${REPO_ROOT}/scripts/common/lib.sh"
|
||||||
|
|
||||||
|
ACTION="${1:-all}"
|
||||||
|
RUN_ID="${RUN_ID:-kimi3-sm120-pr-representative-$(date '+%Y%m%d-%H%M%S')}"
|
||||||
|
RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}"
|
||||||
|
|
||||||
|
MODEL_PATH="${MODEL_PATH:-/data/hf_models/Kimi-K3}"
|
||||||
|
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-kimi-k3}"
|
||||||
|
BASE_IMAGE="${BASE_IMAGE:-lmsysorg/sglang:kimi-k3-fiv617situ-warm}"
|
||||||
|
PR_IMAGE="${PR_IMAGE:-local/sglang:kimi-k3-sm120-pr-300c87a-fi-b525c51}"
|
||||||
|
SGLANG_SOURCE="${SGLANG_SOURCE:-/data/hzy/src/sglang-kimi-sm120-draft-wt}"
|
||||||
|
FLASHINFER_SOURCE="${FLASHINFER_SOURCE:-/data/hzy/src/flashinfer-pr4460-kimi-test-wt}"
|
||||||
|
ARTIFACT_DIR="${ARTIFACT_DIR:-/data/hzy/artifacts/sglang-pr-kimi-sm120-300c87a}"
|
||||||
|
FLASHINFER_WHEEL="${FLASHINFER_WHEEL:-${ARTIFACT_DIR}/flashinfer_python-0.6.18-py3-none-any.whl}"
|
||||||
|
SGLANG_KERNEL_WHEEL="${SGLANG_KERNEL_WHEEL:-${ARTIFACT_DIR}/sglang_kernel-0.4.6.post1-cp310-abi3-manylinux2014_x86_64.whl}"
|
||||||
|
BUILD_CONTEXT="${BUILD_CONTEXT:-/tmp/kimi3_sm120_pr_validation_context}"
|
||||||
|
FLASHINFER_CACHE="${FLASHINFER_CACHE:-/data/hzy/cache/flashinfer-pr-b525c51}"
|
||||||
|
|
||||||
|
HEAD_HOST="${HEAD_HOST:-174.1.60.1}"
|
||||||
|
NODE_SSH_USER="${NODE_SSH_USER:-user}"
|
||||||
|
NODE_HOSTS=(174.1.60.1 174.1.60.2 174.1.60.3 174.1.60.4)
|
||||||
|
SSH_OPTS=(-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10)
|
||||||
|
DIST_PORT="${DIST_PORT:-20000}"
|
||||||
|
PORT="${PORT:-30000}"
|
||||||
|
CONTAINER_PREFIX="kimi3_sm120_pr_validation"
|
||||||
|
|
||||||
|
INPUT_LEN=16384
|
||||||
|
OUTPUT_LEN=1
|
||||||
|
CONCURRENCY=8
|
||||||
|
CHUNKED_PREFILL_SIZE=8192
|
||||||
|
NUM_PROMPTS="${NUM_PROMPTS:-40}"
|
||||||
|
REPEATS="${REPEATS:-3}"
|
||||||
|
WARMUP_REQUESTS="${WARMUP_REQUESTS:-2}"
|
||||||
|
HEALTH_WAIT_S="${HEALTH_WAIT_S:-2400}"
|
||||||
|
BACKENDS=(marlin flashinfer_mxfp4)
|
||||||
|
|
||||||
|
mkdir -p "${RESULT_ROOT}"/{build,service,raw,bench,gpu}
|
||||||
|
log_init "${RESULT_ROOT}/orchestrator.log"
|
||||||
|
|
||||||
|
require_password() {
|
||||||
|
if [[ -z "${SUDO_PASSWORD:-}" && -n "${SUDO_PASSWORD_FILE:-}" ]]; then
|
||||||
|
[[ -r "${SUDO_PASSWORD_FILE}" ]] || {
|
||||||
|
echo "ERROR: cannot read SUDO_PASSWORD_FILE=${SUDO_PASSWORD_FILE}" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
IFS= read -r SUDO_PASSWORD <"${SUDO_PASSWORD_FILE}"
|
||||||
|
fi
|
||||||
|
[[ -n "${SUDO_PASSWORD:-}" ]] || {
|
||||||
|
echo "ERROR: set SUDO_PASSWORD or SUDO_PASSWORD_FILE" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
is_head() { [[ "$1" == "${HEAD_HOST}" ]]; }
|
||||||
|
|
||||||
|
sudo_host() {
|
||||||
|
local host="$1"
|
||||||
|
shift
|
||||||
|
require_password
|
||||||
|
if is_head "$host"; then
|
||||||
|
printf '%s\n' "${SUDO_PASSWORD}" | sudo -S -p '' -- "$@"
|
||||||
|
else
|
||||||
|
local remote_cmd
|
||||||
|
printf -v remote_cmd '%q ' "$@"
|
||||||
|
printf '%s\n' "${SUDO_PASSWORD}" | \
|
||||||
|
ssh "${SSH_OPTS[@]}" "${NODE_SSH_USER}@${host}" \
|
||||||
|
"sudo -S -p '' -- ${remote_cmd}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
check_inputs() {
|
||||||
|
local path
|
||||||
|
for path in "${MODEL_PATH}" "${SGLANG_SOURCE}/python/sglang" \
|
||||||
|
"${FLASHINFER_SOURCE}" "${FLASHINFER_WHEEL}" \
|
||||||
|
"${SGLANG_KERNEL_WHEEL}" \
|
||||||
|
"${SCRIPT_DIR}/Dockerfile.pr_validation"; do
|
||||||
|
[[ -e "$path" ]] || { echo "ERROR: missing ${path}" >&2; exit 2; }
|
||||||
|
done
|
||||||
|
|
||||||
|
SGLANG_COMMIT="$(git -C "${SGLANG_SOURCE}" rev-parse HEAD)"
|
||||||
|
FLASHINFER_COMMIT="$(git -C "${FLASHINFER_SOURCE}" rev-parse HEAD~1)"
|
||||||
|
[[ "${SGLANG_COMMIT}" == 300c87a431ac40d3e7817246376b7fe20932db09 ]] || {
|
||||||
|
echo "ERROR: unexpected SGLang commit ${SGLANG_COMMIT}" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
[[ "${FLASHINFER_COMMIT}" == b525c51* ]] || {
|
||||||
|
echo "ERROR: unexpected FlashInfer implementation commit ${FLASHINFER_COMMIT}" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
export SGLANG_COMMIT FLASHINFER_COMMIT
|
||||||
|
}
|
||||||
|
|
||||||
|
prepare_build_context() {
|
||||||
|
log "preparing exact Draft build context"
|
||||||
|
rm -rf "${BUILD_CONTEXT}"
|
||||||
|
mkdir -p "${BUILD_CONTEXT}/sglang"
|
||||||
|
cp "${SCRIPT_DIR}/Dockerfile.pr_validation" "${BUILD_CONTEXT}/Dockerfile"
|
||||||
|
cp "${FLASHINFER_WHEEL}" "${BUILD_CONTEXT}/"
|
||||||
|
cp "${SGLANG_KERNEL_WHEEL}" "${BUILD_CONTEXT}/"
|
||||||
|
cp "${SGLANG_SOURCE}/test/registered/unit/layers/quantization/test_mxfp4_sm120_cutlass.py" \
|
||||||
|
"${BUILD_CONTEXT}/test_mxfp4_sm120_cutlass.py"
|
||||||
|
tar -C "${SGLANG_SOURCE}/python/sglang" -cf - . | \
|
||||||
|
tar -C "${BUILD_CONTEXT}/sglang" -xf -
|
||||||
|
{
|
||||||
|
printf 'sglang=%s\n' "${SGLANG_COMMIT}"
|
||||||
|
printf 'flashinfer_implementation=%s\n' "${FLASHINFER_COMMIT}"
|
||||||
|
sha256sum "${FLASHINFER_WHEEL}" "${SGLANG_KERNEL_WHEEL}"
|
||||||
|
} >"${BUILD_CONTEXT}/source_identity.txt"
|
||||||
|
cp "${BUILD_CONTEXT}/source_identity.txt" "${RESULT_ROOT}/build/"
|
||||||
|
du -sh "${BUILD_CONTEXT}" | tee "${RESULT_ROOT}/build/context_size.txt"
|
||||||
|
}
|
||||||
|
|
||||||
|
stage_context() {
|
||||||
|
local host="$1"
|
||||||
|
is_head "$host" && return
|
||||||
|
ssh "${SSH_OPTS[@]}" "${NODE_SSH_USER}@${host}" \
|
||||||
|
"rm -rf '${BUILD_CONTEXT}' && mkdir -p '${BUILD_CONTEXT}'"
|
||||||
|
tar -C "${BUILD_CONTEXT}" -cf - . | \
|
||||||
|
ssh "${SSH_OPTS[@]}" "${NODE_SSH_USER}@${host}" \
|
||||||
|
"tar -C '${BUILD_CONTEXT}' -xf -"
|
||||||
|
}
|
||||||
|
|
||||||
|
build_one_node() {
|
||||||
|
local host="$1"
|
||||||
|
sudo_host "$host" docker build \
|
||||||
|
--build-arg "BASE_IMAGE=${BASE_IMAGE}" \
|
||||||
|
--build-arg "SGLANG_COMMIT=${SGLANG_COMMIT}" \
|
||||||
|
--build-arg "FLASHINFER_COMMIT=${FLASHINFER_COMMIT}" \
|
||||||
|
--tag "${PR_IMAGE}" "${BUILD_CONTEXT}" \
|
||||||
|
>"${RESULT_ROOT}/build/${host}.log" 2>&1
|
||||||
|
sudo_host "$host" docker image inspect "${PR_IMAGE}" \
|
||||||
|
--format '{{.Id}} {{.Size}} {{json .Config.Labels}}' \
|
||||||
|
>"${RESULT_ROOT}/build/${host}.image.txt"
|
||||||
|
sudo_host "$host" docker run --rm --entrypoint python3 "${PR_IMAGE}" -c \
|
||||||
|
"import importlib.metadata as m; print('sglang=' + m.version('sglang')); print('sglang-kernel=' + m.version('sglang-kernel')); print('flashinfer-python=' + m.version('flashinfer-python')); print(open('/opt/pr-build/source_identity.txt').read(), end='')" \
|
||||||
|
>"${RESULT_ROOT}/build/${host}.packages.txt"
|
||||||
|
}
|
||||||
|
|
||||||
|
build_all_nodes() {
|
||||||
|
check_inputs
|
||||||
|
prepare_build_context
|
||||||
|
local host pid rc=0
|
||||||
|
for host in "${NODE_HOSTS[@]}"; do stage_context "$host"; done
|
||||||
|
local -a pids=()
|
||||||
|
for host in "${NODE_HOSTS[@]}"; do
|
||||||
|
build_one_node "$host" &
|
||||||
|
pids+=("$!")
|
||||||
|
done
|
||||||
|
for pid in "${pids[@]}"; do wait "$pid" || rc=1; done
|
||||||
|
(( rc == 0 )) || { log "ERROR: image build failed"; return 1; }
|
||||||
|
log "exact Draft image built on all nodes"
|
||||||
|
}
|
||||||
|
|
||||||
|
prewarm_one_node() {
|
||||||
|
local host="$1"
|
||||||
|
sudo_host "$host" mkdir -p "${FLASHINFER_CACHE}"
|
||||||
|
sudo_host "$host" docker run --rm --gpus device=0 \
|
||||||
|
-v "${FLASHINFER_CACHE}:/root/.cache/flashinfer" \
|
||||||
|
-e FLASHINFER_DISABLE_VERSION_CHECK=1 \
|
||||||
|
--entrypoint python3 "${PR_IMAGE}" -m pytest -q -s \
|
||||||
|
/opt/pr-tests/test_mxfp4_sm120_cutlass.py \
|
||||||
|
-k kimi_k3_sm120_situ_layout_and_noncontiguous_input \
|
||||||
|
>"${RESULT_ROOT}/build/${host}.prewarm.log" 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
prewarm_all_nodes() {
|
||||||
|
local host pid rc=0
|
||||||
|
local -a pids=()
|
||||||
|
for host in "${NODE_HOSTS[@]}"; do
|
||||||
|
prewarm_one_node "$host" &
|
||||||
|
pids+=("$!")
|
||||||
|
done
|
||||||
|
for pid in "${pids[@]}"; do wait "$pid" || rc=1; done
|
||||||
|
(( rc == 0 )) || { log "ERROR: FlashInfer prewarm failed"; return 1; }
|
||||||
|
log "exact #4460 kernel prewarmed on all nodes"
|
||||||
|
}
|
||||||
|
|
||||||
|
container_name() { printf '%s_node%s' "${CONTAINER_PREFIX}" "$1"; }
|
||||||
|
|
||||||
|
stop_service() {
|
||||||
|
local rank host
|
||||||
|
for rank in 0 1 2 3; do
|
||||||
|
host="${NODE_HOSTS[$rank]}"
|
||||||
|
sudo_host "$host" docker rm -f "$(container_name "$rank")" \
|
||||||
|
>/dev/null 2>&1 || true
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
collect_service_logs() {
|
||||||
|
local label="$1" rank host
|
||||||
|
for rank in 0 1 2 3; do
|
||||||
|
host="${NODE_HOSTS[$rank]}"
|
||||||
|
sudo_host "$host" docker logs "$(container_name "$rank")" \
|
||||||
|
>"${RESULT_ROOT}/service/${label}_node${rank}.log" 2>&1 || true
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
collect_gpu() {
|
||||||
|
local label="$1" rank host
|
||||||
|
for rank in 0 1 2 3; do
|
||||||
|
host="${NODE_HOSTS[$rank]}"
|
||||||
|
sudo_host "$host" nvidia-smi \
|
||||||
|
--query-gpu=timestamp,index,memory.used,memory.total,utilization.gpu,power.draw \
|
||||||
|
--format=csv,noheader,nounits \
|
||||||
|
>"${RESULT_ROOT}/gpu/${label}_node${rank}.csv" 2>&1 || true
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
verify_service_logs() {
|
||||||
|
local label="$1"
|
||||||
|
local pattern='CUDA out of memory|torch\.OutOfMemoryError|Traceback|EngineDeadError|NCCL[^[:cntrl:]]*(error|failed)|connection refused|Terminated'
|
||||||
|
if grep -Ein "${pattern}" "${RESULT_ROOT}/service/${label}_node"*.log \
|
||||||
|
>"${RESULT_ROOT}/service/${label}_fatal_scan.txt"; then
|
||||||
|
log "ERROR: fatal pattern found in service logs label=${label}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
: >"${RESULT_ROOT}/service/${label}_fatal_scan.txt"
|
||||||
|
}
|
||||||
|
|
||||||
|
start_node() {
|
||||||
|
local rank="$1" backend="$2"
|
||||||
|
local host="${NODE_HOSTS[$rank]}" name bootstrap
|
||||||
|
name="$(container_name "$rank")"
|
||||||
|
bootstrap="export SGLANG_HOST_IP=174.1.60.$((rank + 1)); exec python3 -m sglang.launch_server --model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size 32 --ep-size 4 --nnodes 4 --node-rank ${rank} --dist-init-addr ${HEAD_HOST}:${DIST_PORT} --trust-remote-code --moe-runner-backend ${backend} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --host 0.0.0.0 --port ${PORT}"
|
||||||
|
local -a cmd=(
|
||||||
|
docker run -d --name "$name"
|
||||||
|
--gpus all --network host --ipc=host --ulimit memlock=-1
|
||||||
|
--device /dev/infiniband --shm-size 32g --entrypoint bash
|
||||||
|
-v "${MODEL_PATH}:${MODEL_PATH}:ro"
|
||||||
|
-v "${FLASHINFER_CACHE}:/root/.cache/flashinfer"
|
||||||
|
-e CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
||||||
|
-e NCCL_SOCKET_IFNAME=bond0 -e GLOO_SOCKET_IFNAME=bond0
|
||||||
|
-e NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3
|
||||||
|
-e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7
|
||||||
|
-e NCCL_CUMEM_ENABLE=1 -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||||
|
-e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1
|
||||||
|
-e FLASHINFER_DISABLE_VERSION_CHECK=1
|
||||||
|
"${PR_IMAGE}" -lc "$bootstrap"
|
||||||
|
)
|
||||||
|
printf '%q ' "${cmd[@]}" \
|
||||||
|
>"${RESULT_ROOT}/service/${backend}_node${rank}.cmd.txt"
|
||||||
|
printf '\n' >>"${RESULT_ROOT}/service/${backend}_node${rank}.cmd.txt"
|
||||||
|
sudo_host "$host" "${cmd[@]}" >/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_health() {
|
||||||
|
local backend="$1" i
|
||||||
|
for ((i = 1; i <= HEALTH_WAIT_S; i++)); do
|
||||||
|
if curl --fail --silent --max-time 5 \
|
||||||
|
"http://${HEAD_HOST}:${PORT}/health" >/dev/null 2>&1; then
|
||||||
|
log "service healthy backend=${backend} wait_s=${i}"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if (( i % 30 == 0 )); then
|
||||||
|
log "waiting backend=${backend} elapsed_s=${i}"
|
||||||
|
collect_service_logs "${backend}_starting"
|
||||||
|
if grep -Eiq 'Traceback|CUDA out of memory|NCCL.*(error|failed)|EngineDeadError' \
|
||||||
|
"${RESULT_ROOT}/service/${backend}_starting_node"*.log; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
start_service() {
|
||||||
|
local backend="$1"
|
||||||
|
stop_service
|
||||||
|
log "starting TP32/EP4 backend=${backend} chunk=8192"
|
||||||
|
start_node 1 "$backend"
|
||||||
|
start_node 2 "$backend"
|
||||||
|
start_node 3 "$backend"
|
||||||
|
sleep 5
|
||||||
|
start_node 0 "$backend"
|
||||||
|
wait_health "$backend" || {
|
||||||
|
collect_service_logs "${backend}_startup_failed"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
collect_service_logs "${backend}_healthy"
|
||||||
|
collect_gpu "${backend}_healthy"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_bench() {
|
||||||
|
local backend="$1" repeat="$2"
|
||||||
|
local stem="${backend}_chunk8192_c8_r${repeat}"
|
||||||
|
local output="${RESULT_ROOT}/raw/${stem}.jsonl"
|
||||||
|
rm -f "$output"
|
||||||
|
log "bench backend=${backend} repeat=${repeat}/${REPEATS}"
|
||||||
|
sudo_host "${HEAD_HOST}" docker run --rm --network host \
|
||||||
|
-v "${MODEL_PATH}:${MODEL_PATH}:ro" \
|
||||||
|
-v "${RESULT_ROOT}:${RESULT_ROOT}" \
|
||||||
|
-e PYTHONUNBUFFERED=1 --entrypoint python3 "${PR_IMAGE}" \
|
||||||
|
-m sglang.benchmark.serving \
|
||||||
|
--backend sglang --host "${HEAD_HOST}" --port "${PORT}" \
|
||||||
|
--tokenizer "${MODEL_PATH}" --dataset-name random-ids \
|
||||||
|
--random-input-len "${INPUT_LEN}" --random-output-len "${OUTPUT_LEN}" \
|
||||||
|
--random-range-ratio 1.0 --num-prompts "${NUM_PROMPTS}" \
|
||||||
|
--max-concurrency "${CONCURRENCY}" --request-rate 10000 \
|
||||||
|
--warmup-requests "${WARMUP_REQUESTS}" --output-file "$output" \
|
||||||
|
--output-details --disable-tqdm \
|
||||||
|
>"${RESULT_ROOT}/bench/${stem}.log" 2>&1
|
||||||
|
python3 - "$output" "${NUM_PROMPTS}" <<'PY'
|
||||||
|
import json, sys
|
||||||
|
path, expected = sys.argv[1], int(sys.argv[2])
|
||||||
|
rows = [json.loads(x) for x in open(path, encoding="utf-8") if x.strip()]
|
||||||
|
assert len(rows) == 1, (path, len(rows))
|
||||||
|
assert rows[0].get("completed") == expected, rows[0].get("completed")
|
||||||
|
assert not rows[0].get("errors"), rows[0].get("errors")
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
run_ab() {
|
||||||
|
local backend repeat
|
||||||
|
collect_gpu before
|
||||||
|
for backend in "${BACKENDS[@]}"; do
|
||||||
|
start_service "$backend"
|
||||||
|
for ((repeat = 1; repeat <= REPEATS; repeat++)); do
|
||||||
|
run_bench "$backend" "$repeat"
|
||||||
|
done
|
||||||
|
collect_service_logs "${backend}_completed"
|
||||||
|
verify_service_logs "${backend}_completed"
|
||||||
|
collect_gpu "${backend}_completed"
|
||||||
|
stop_service
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
collect_gpu after
|
||||||
|
}
|
||||||
|
|
||||||
|
summarize() {
|
||||||
|
python3 - "${RESULT_ROOT}" <<'PY'
|
||||||
|
import csv, json, re, statistics, sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
root = Path(sys.argv[1])
|
||||||
|
pat = re.compile(r"(.+)_chunk8192_c8_r(\d+)\.jsonl$")
|
||||||
|
rows = []
|
||||||
|
for path in sorted((root / "raw").glob("*.jsonl")):
|
||||||
|
m = pat.match(path.name)
|
||||||
|
if not m:
|
||||||
|
continue
|
||||||
|
data = next(json.loads(x) for x in path.read_text().splitlines() if x.strip())
|
||||||
|
rows.append({"backend": m.group(1), "repeat": int(m.group(2)), **data})
|
||||||
|
|
||||||
|
metrics = ["request_throughput", "input_throughput", "total_throughput",
|
||||||
|
"median_ttft_ms", "p95_ttft_ms", "median_e2e_latency_ms"]
|
||||||
|
summary = []
|
||||||
|
for backend in ("marlin", "flashinfer_mxfp4"):
|
||||||
|
group = [x for x in rows if x["backend"] == backend]
|
||||||
|
if len(group) != 3:
|
||||||
|
raise SystemExit(f"expected 3 repeats for {backend}, got {len(group)}")
|
||||||
|
item = {"backend": backend, "repeats": len(group),
|
||||||
|
"completed_each": [x.get("completed") for x in group]}
|
||||||
|
for metric in metrics:
|
||||||
|
item[f"median_{metric}"] = statistics.median(float(x[metric]) for x in group)
|
||||||
|
summary.append(item)
|
||||||
|
idx = {x["backend"]: x for x in summary}
|
||||||
|
base, cand = idx["marlin"], idx["flashinfer_mxfp4"]
|
||||||
|
comparison = {
|
||||||
|
"input_throughput_change_pct":
|
||||||
|
(cand["median_input_throughput"] / base["median_input_throughput"] - 1) * 100,
|
||||||
|
"median_ttft_change_pct":
|
||||||
|
(cand["median_median_ttft_ms"] / base["median_median_ttft_ms"] - 1) * 100,
|
||||||
|
"p95_ttft_change_pct":
|
||||||
|
(cand["median_p95_ttft_ms"] / base["median_p95_ttft_ms"] - 1) * 100,
|
||||||
|
}
|
||||||
|
payload = {
|
||||||
|
"run_id": root.name,
|
||||||
|
"shape": {"input_len": 16384, "output_len": 1,
|
||||||
|
"concurrency": 8, "chunked_prefill_size": 8192,
|
||||||
|
"tp": 32, "ep": 4},
|
||||||
|
"summary": summary,
|
||||||
|
"comparison": comparison,
|
||||||
|
}
|
||||||
|
(root / "summary.json").write_text(json.dumps(payload, indent=2))
|
||||||
|
with (root / "results.csv").open("w", newline="") as f:
|
||||||
|
fields = ["backend", "repeat", "completed", *metrics]
|
||||||
|
w = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore")
|
||||||
|
w.writeheader(); w.writerows(rows)
|
||||||
|
print(json.dumps(payload, indent=2))
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
collect_service_logs cleanup 2>/dev/null || true
|
||||||
|
stop_service 2>/dev/null || true
|
||||||
|
}
|
||||||
|
trap cleanup EXIT INT TERM
|
||||||
|
|
||||||
|
case "${ACTION}" in
|
||||||
|
build)
|
||||||
|
require_password; check_inputs; build_all_nodes; prewarm_all_nodes
|
||||||
|
;;
|
||||||
|
run)
|
||||||
|
require_password; check_inputs; run_ab; summarize
|
||||||
|
;;
|
||||||
|
all)
|
||||||
|
require_password; check_inputs; build_all_nodes; prewarm_all_nodes
|
||||||
|
run_ab; summarize
|
||||||
|
;;
|
||||||
|
summarize)
|
||||||
|
summarize
|
||||||
|
;;
|
||||||
|
stop)
|
||||||
|
require_password; stop_service
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Usage: $0 {all|build|run|summarize|stop}" >&2
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
Loading…
x
Reference in New Issue
Block a user