144 lines
4.8 KiB
Bash
Executable File
144 lines
4.8 KiB
Bash
Executable File
#!/bin/bash
|
|
# Max throughput benchmark for sglang 8-card DeepSeek-V4-Flash
|
|
set -e
|
|
|
|
cd /data/user1/yy
|
|
mkdir -p bench_results logs
|
|
|
|
PYTHON="/data/user1/yy/envs/sglang/bin/python"
|
|
BENCH="-m sglang.bench_serving"
|
|
HOST="127.0.0.1"
|
|
PORT="30000"
|
|
MODEL="/data/models/DeepSeek-V4-Flash"
|
|
DATASET="/data/user1/yy/datasets/ShareGPT_V4.3_unfiltered_cleaned_split.json"
|
|
NUM_PROMPTS=2000
|
|
SEED=42
|
|
|
|
RESULT_DIR="bench_results/sglang_8card_max_throughput_$(date +%Y%m%d_%H%M%S)"
|
|
mkdir -p "${RESULT_DIR}"
|
|
|
|
SUMMARY_FILE="${RESULT_DIR}/summary.json"
|
|
LOG_FILE="${RESULT_DIR}/benchmark.log"
|
|
SUMMARY_TMP="${RESULT_DIR}/.summary_tmp.json"
|
|
echo '[]' > "${SUMMARY_TMP}"
|
|
|
|
log() {
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${LOG_FILE}"
|
|
}
|
|
|
|
wait_for_server() {
|
|
log "Waiting for sglang server at ${HOST}:${PORT} to become ready..."
|
|
local retries=0
|
|
local max_retries=120
|
|
while ! curl -s "http://${HOST}:${PORT}/v1/models" > /dev/null 2>&1; do
|
|
retries=$((retries + 1))
|
|
if [ "${retries}" -ge "${max_retries}" ]; then
|
|
log "ERROR: Server not ready after ${max_retries} retries"
|
|
exit 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
log "Server is ready."
|
|
}
|
|
|
|
run_bench() {
|
|
local name="$1"
|
|
shift
|
|
local output_file="${RESULT_DIR}/${name}.json"
|
|
local detail_log="${RESULT_DIR}/${name}.log"
|
|
|
|
log "---------------------------------------------------"
|
|
log "Running benchmark: ${name}"
|
|
log "Args: $*"
|
|
log "Output: ${output_file}"
|
|
|
|
${PYTHON} ${BENCH} \
|
|
--backend sglang \
|
|
--host "${HOST}" \
|
|
--port "${PORT}" \
|
|
--dataset-name sharegpt \
|
|
--dataset-path "${DATASET}" \
|
|
--num-prompts "${NUM_PROMPTS}" \
|
|
--model "${MODEL}" \
|
|
--output-file "${output_file}" \
|
|
--output-details \
|
|
--seed "${SEED}" \
|
|
"$@" \
|
|
> "${detail_log}" 2>&1
|
|
|
|
local status=$?
|
|
if [ ${status} -ne 0 ]; then
|
|
log "ERROR: Benchmark ${name} failed with exit code ${status}"
|
|
return ${status}
|
|
fi
|
|
|
|
log "Benchmark ${name} completed successfully."
|
|
|
|
local req_throughput=$(grep -oP 'Request throughput \(req/s\):\s+\K[0-9.]+' "${detail_log}" || echo "null")
|
|
local input_throughput=$(grep -oP 'Input token throughput \(tok/s\):\s+\K[0-9.]+' "${detail_log}" || echo "null")
|
|
local output_throughput=$(grep -oP 'Output token throughput \(tok/s\):\s+\K[0-9.]+' "${detail_log}" || echo "null")
|
|
local total_throughput=$(grep -oP 'Total token throughput \(tok/s\):\s+\K[0-9.]+' "${detail_log}" || echo "null")
|
|
local mean_ttft=$(grep -oP 'Mean TTFT \(ms\):\s+\K[0-9.]+' "${detail_log}" || echo "null")
|
|
local mean_tpot=$(grep -oP 'Mean TPOT \(ms\):\s+\K[0-9.]+' "${detail_log}" || echo "null")
|
|
local mean_e2e=$(grep -oP 'Mean E2E Latency \(ms\):\s+\K[0-9.]+' "${detail_log}" || echo "null")
|
|
local duration=$(grep -oP 'Benchmark duration \(s\):\s+\K[0-9.]+' "${detail_log}" || echo "null")
|
|
local accept_length=$(grep -oP 'Accept length:\s+\K[0-9.]+' "${detail_log}" || echo "null")
|
|
|
|
${PYTHON} - <<PYEOF
|
|
import json
|
|
entry = {
|
|
"name": "${name}",
|
|
"args": "$*",
|
|
"duration_s": ${duration},
|
|
"request_throughput": ${req_throughput},
|
|
"input_token_throughput": ${input_throughput},
|
|
"output_token_throughput": ${output_throughput},
|
|
"total_token_throughput": ${total_throughput},
|
|
"mean_ttft_ms": ${mean_ttft},
|
|
"mean_tpot_ms": ${mean_tpot},
|
|
"mean_e2e_latency_ms": ${mean_e2e},
|
|
"accept_length": ${accept_length},
|
|
"output_file": "${output_file}"
|
|
}
|
|
with open("${SUMMARY_TMP}", "r") as f:
|
|
data = json.load(f)
|
|
data.append(entry)
|
|
with open("${SUMMARY_TMP}", "w") as f:
|
|
json.dump(data, f, indent=2)
|
|
PYEOF
|
|
}
|
|
|
|
main() {
|
|
log "Starting max-throughput benchmark for sglang 8-card DeepSeek-V4-Flash"
|
|
log "Model: ${MODEL}"
|
|
log "Dataset: ${DATASET}"
|
|
log "Num prompts per test: ${NUM_PROMPTS}"
|
|
log "Results will be saved to: ${RESULT_DIR}"
|
|
|
|
wait_for_server
|
|
|
|
log "=== Max concurrency sweep ==="
|
|
for c in 128 256 512 1024; do
|
|
run_bench "sharegpt_concurrency_${c}" --max-concurrency "${c}"
|
|
done
|
|
|
|
mv "${SUMMARY_TMP}" "${SUMMARY_FILE}"
|
|
|
|
log "=== All max-throughput benchmarks completed ==="
|
|
log "Summary saved to: ${SUMMARY_FILE}"
|
|
log "Detailed results in: ${RESULT_DIR}"
|
|
|
|
${PYTHON} - <<PYEOF
|
|
import json
|
|
with open("${SUMMARY_FILE}", "r") as f:
|
|
data = json.load(f)
|
|
print("\n============ Max Throughput Summary ============")
|
|
print(f"{'Name':<30} {'Req/s':>8} {'In tok/s':>10} {'Out tok/s':>10} {'Total tok/s':>12} {'TTFT':>8} {'TPOT':>8} {'E2E':>10}")
|
|
print("-" * 110)
|
|
for entry in data:
|
|
print(f"{entry['name']:<30} {entry['request_throughput']:>8.2f} {entry['input_token_throughput']:>10.2f} {entry['output_token_throughput']:>10.2f} {entry['total_token_throughput']:>12.2f} {entry['mean_ttft_ms']:>8.1f} {entry['mean_tpot_ms']:>8.1f} {entry['mean_e2e_latency_ms']:>10.1f}")
|
|
PYEOF
|
|
}
|
|
|
|
main "$@"
|