diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py b/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py index 8bf26df..bdd3224 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py @@ -6,6 +6,7 @@ Usage: """ import argparse import json +import re from collections import defaultdict from pathlib import Path @@ -106,8 +107,8 @@ def main(): f.write("| " + " | ".join(headers) + " |\n") f.write("|" + "|".join(["---"] * len(headers)) + "|\n") - for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, x.replace("c", "").replace("i", "_").replace("o", "_").split("_")[1:]))): - cfg_part, isl, dsl = scenario_name.replace("c", " ").replace("i", " ").replace("o", " ").split() + for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, re.findall(r"\d+", x)))): + _, isl, dsl = re.findall(r"\d+", scenario_name) # cfg_part not used; just for readability. for label, data in configs: s = by_scenario[scenario_name].get(label) @@ -134,7 +135,7 @@ def main(): f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n") best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {} for scenario_name, backends in by_scenario.items(): - _, isl, dsl = scenario_name.replace("c", " ").replace("i", " ").replace("o", " ").split() + _, isl, dsl = re.findall(r"\d+", scenario_name) isl_i, dsl_i = int(isl), int(dsl) for label, s in backends.items(): m = s["metrics"] diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env index a0aa5ee..1652629 100644 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env @@ -46,3 +46,8 @@ DRY_RUN="${DRY_RUN:-0}" # Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios. GRID_LIMIT="${GRID_LIMIT:-0}" + +# Docker deployment switch. If 1, vLLM is launched via Docker using the image +# below instead of the local VENV_VLLM environment. +USE_DOCKER="${USE_DOCKER:-0}" +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh index ebd3b85..99e8de3 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh @@ -204,7 +204,12 @@ append_scenario_record() { scenario_json="$("$PYTHON" -c " import json, sys pairs = [a.split('=', 1) for a in sys.argv[1:]] -d = {k: json.loads(v) for k, v in pairs} +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v print(json.dumps(d, ensure_ascii=False)) " "$@")" PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json" @@ -218,7 +223,12 @@ record_skipped_csv() { row="$("$PYTHON" -c " import csv, json, sys, io pairs = [a.split('=', 1) for a in sys.argv[1:]] -d = {k: json.loads(v) for k, v in pairs} +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v buf = io.StringIO() writer = csv.DictWriter(buf, fieldnames=['engine','tp','dp','mark','isl','dsl','concurrency','status','reason','detail_log'], extrasaction='ignore') writer.writerow(d) @@ -358,9 +368,9 @@ run_parallel_config() { fi sname="c${conc}_i${isl}_o${dsl}" - output_file="${raw_dir}/vllm_main_${sname}.jsonl" + output_file="${raw_dir}/vllm_main_${conc}_${isl}_${dsl}.jsonl" detail_log="${phase_log_dir}/vllm_${config_label}_${sname}.log" - gpu_csv="${gpu_log_dir}/gpu_mem_${sname}.csv" + gpu_csv="${gpu_log_dir}/gpu_mem_${conc}_${isl}_${dsl}.csv" if scenario_already_completed "$output_file" "$num" || scenario_already_processed "$result_root" "$sname"; then log "skipping already-processed ${config_label} scenario: ${sname}" @@ -392,6 +402,11 @@ run_parallel_config() { if [[ "$bench_rc" -eq 0 ]]; then log "finished ${config_label} scenario: output=${output_file}" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"completed\"" \ + "note=\"benchmark finished successfully\"" continue fi diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh new file mode 100755 index 0000000..ae7f728 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Start vLLM server in Docker for a given TPxDP configuration. +# Usage: start_vllm_docker.sh +# +# Uses the vllm/vllm-openai image and the same minimal argument set as the +# bare-metal start script. The container is removed automatically on stop. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +cd /data/user1/yy +mkdir -p logs + +IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +PORT="${VLLM_PORT:-30030}" +NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}" +PID_FILE="/data/user1/yy/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" + +LOG="/data/user1/yy/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +# Clean up any stale container with the same name. +docker rm -f "$NAME" >/dev/null 2>&1 || true + +SERVER_ARGS=( + serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype bfloat16 + --block-size 256 + --tensor-parallel-size "$TP" + --no-enable-flashinfer-autotune + --port "$PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --data-parallel-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting vLLM server in Docker (TP=${TP}, DP=${DP}) ===" +echo "Image: $IMAGE" +echo "Model: $MODEL_PATH" +echo "Container name: $NAME" +echo "Host port: $PORT" +echo "Command: vllm $SERVER_ARGS_STR" +echo "Log: $LOG" + +# Run docker in the foreground so that killing the host process stops the +# container (the --rm flag ensures cleanup). nohup lets us background it and +# capture the host PID in the same way as the bare-metal start script. +nohup docker run --rm \ + --name "$NAME" \ + --gpus all \ + --entrypoint vllm \ + -p "${PORT}:${PORT}" \ + -v /data/models:/data/models:ro \ + -v /data/user1/yy/tmp:/tmp \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + "$IMAGE" \ + "${SERVER_ARGS[@]}" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: Docker vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: Docker vLLM server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh index b7ec14b..77b662a 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -19,6 +19,10 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" +if [[ "${USE_DOCKER:-0}" == "1" ]]; then + exec "${SCRIPT_DIR}/start_vllm_docker.sh" "$@" +fi + cd /data/user1/yy mkdir -p logs