From 0aee540c67fcbc7a60367acdc12e746f6b2edd84 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Thu, 9 Jul 2026 10:50:59 +0000 Subject: [PATCH] dsv4_h200_vllm_tp_dp_matrix: single server per config, OOM/restart handling, DRY_RUN/GRID_LIMIT, include 1M --- .../dsv4_h200_vllm_tp_dp_matrix/config.env | 7 + .../dsv4_h200_vllm_tp_dp_matrix/matrix.json | 4 +- .../dsv4_h200_vllm_tp_dp_matrix/run_bench.sh | 250 +++++++++++++----- 3 files changed, 197 insertions(+), 64 deletions(-) diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env index 60a147d..a0aa5ee 100644 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env @@ -39,3 +39,10 @@ SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" # GPU memory sampling interval (seconds). GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Dry-run mode: if 1, only log the server args and scenario plan without starting +# any server or sending requests. +DRY_RUN="${DRY_RUN:-0}" + +# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json b/experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json index 1ed8e9f..539a577 100644 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json @@ -60,8 +60,8 @@ "4096": "N" }, "1048576": { - "128": "N", - "256": "N", + "128": "Y", + "256": "P", "512": "N", "1024": "N", "2048": "N", diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh index e3cdaa1..ebd3b85 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh @@ -18,6 +18,8 @@ MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" MATRIX_MODE="${MATRIX_MODE:-Y}" SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" +DRY_RUN="${DRY_RUN:-0}" +GRID_LIMIT="${GRID_LIMIT:-0}" PYTHON="${VENV_CLIENT}/bin/python" @@ -26,7 +28,7 @@ mkdir -p "$log_dir_global" log_init "${log_dir_global}/orchestrator.log" log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}" -log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE}" +log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE} dry_run=${DRY_RUN} grid_limit=${GRID_LIMIT}" # --------------------------------------------------------------------------- # Helpers @@ -95,13 +97,20 @@ start_server() { log "vllm server tp=${tp} dp=${dp} is healthy on port ${VLLM_PORT}" } -run_warmup() { +restart_server() { local tp="$1" local dp="$2" - local input_len="$3" - local output_len="$4" + log "restarting vllm server tp=${tp} dp=${dp} after non-OOM failure" + stop_server "$tp" "$dp" + sleep 10 + start_server "$tp" "$dp" +} - log "warming up tp=${tp} dp=${dp} (input=${input_len}, output=${output_len}, num=1)" +run_warmup() { + local input_len="$1" + local output_len="$2" + + log "warming up (input=${input_len}, output=${output_len}, num=1)" "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \ --backend vllm \ --host 127.0.0.1 \ @@ -110,7 +119,7 @@ run_warmup() { --output-len "$output_len" \ --num 1 \ --env-python "$PYTHON" \ - >> "${log_dir_global}/vllm_tp${tp}_dp${dp}.warmup.log" 2>&1 + >> "${log_dir_global}/warmup.log" 2>&1 log "warmup completed" } @@ -136,6 +145,37 @@ except Exception: [[ "${completed:-0}" -ge "$expected" ]] } +scenario_already_processed() { + local result_root="$1" + local scenario_name="$2" + local json_path="${result_root}/results.json" + [[ -f "$json_path" ]] || return 1 + "$PYTHON" -c " +import json, sys +path, name = sys.argv[1], sys.argv[2] +try: + with open(path, 'r', encoding='utf-8') as f: + data = json.load(f) + for s in data.get('scenarios', []): + if s.get('name') == name: + if s.get('status') or s.get('metrics', {}).get('success', 0) > 0: + sys.exit(0) +except Exception: + pass +sys.exit(1) +" "$json_path" "$scenario_name" +} + +detect_oom() { + local detail_log="$1" + local server_outer_log="$2" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + if grep -Eiq "$pattern" "$detail_log" "$server_outer_log" 2>/dev/null; then + return 0 + fi + return 1 +} + start_gpu_monitor() { local csv_path="$1" mkdir -p "$(dirname "$csv_path")" @@ -160,7 +200,6 @@ append_scenario_record() { local result_root="$1" local json_path="$result_root/results.json" shift - # Remaining args are key=value pairs. local scenario_json scenario_json="$("$PYTHON" -c " import json, sys @@ -171,6 +210,54 @@ print(json.dumps(d, ensure_ascii=False)) PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json" } +record_skipped_csv() { + local csv_path="$1" + shift + # Args: key=value + local row + row="$("$PYTHON" -c " +import csv, json, sys, io +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {k: json.loads(v) for k, v in pairs} +buf = io.StringIO() +writer = csv.DictWriter(buf, fieldnames=['engine','tp','dp','mark','isl','dsl','concurrency','status','reason','detail_log'], extrasaction='ignore') +writer.writerow(d) +print(buf.getvalue().strip()) +" "$@")" + echo "$row" >> "$csv_path" +} + +skip_remaining_scenarios() { + local result_root="$1" + local scenario_tsv="$2" + local start_index="$3" + local status="$4" + local reason="$5" + local tp="$6" + local dp="$7" + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + + local i=0 + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do + if (( i < start_index )); then + i=$((i + 1)) + continue + fi + i=$((i + 1)) + local sname="c${conc}_i${isl}_o${dsl}" + if scenario_already_processed "$result_root" "$sname"; then + continue + fi + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"${status}\"" \ + "note=\"${reason}\"" + record_skipped_csv "$skipped_csv" \ + "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=${status}" "reason=${reason}" + done +} + # --------------------------------------------------------------------------- # Per-configuration runner # --------------------------------------------------------------------------- @@ -197,11 +284,6 @@ run_parallel_config() { total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)" log "generated ${total_scenarios} scenarios for ${config_label}" - if [[ "$total_scenarios" -eq 0 ]]; then - log "no scenarios for ${config_label}; skipping" - return 0 - fi - # Write metadata. ensure_result_root "$result_root" write_metadata_json \ @@ -218,7 +300,6 @@ run_parallel_config() { "$VENV_VLLM" \ "H200 vLLM TP×DP matrix for DeepSeek-V4-Flash" - # Embed static config and the exact server command line used. local server_args_str server_args_str="$(build_server_args "$tp" "$dp")" jq --arg tp "$tp" --arg dp "$dp" --arg cuda "$CUDA_VISIBLE_DEVICES" --arg args "$server_args_str" \ @@ -232,43 +313,57 @@ run_parallel_config() { }' "${result_root}/results.json" > "${result_root}/results.json.tmp" && \ mv "${result_root}/results.json.tmp" "${result_root}/results.json" - # Group scenarios by input_len. We restart the server for each ISL to keep - # memory state isolated and to warm up per context length. - local current_isl="" - local group_started=false + if [[ "$DRY_RUN" == "1" ]]; then + log "DRY_RUN: would start server with args: ${server_args_str}" + local line + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do + log "DRY_RUN: ${config_label} scenario mark=${mark} c=${conc} i=${isl} o=${dsl} n=${num}" + done + log "===== ${config_label} DONE (dry run) =====" + return 0 + fi - tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do - # When the input length changes, restart the server for the new group. - if [[ "$isl" != "$current_isl" ]]; then - if [[ "$group_started" == true ]]; then - stop_server "$tp" "$dp" - fi - current_isl="$isl" + # Initialize skipped_after_oom.csv for this run. + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + if [[ ! -f "$skipped_csv" ]]; then + echo "engine,tp,dp,mark,isl,dsl,concurrency,status,reason,detail_log" > "$skipped_csv" + fi - stop_server "$tp" "$dp" - if ! start_server "$tp" "$dp"; then - log "ERROR: ${config_label} failed to start for ISL=${isl}; skipping this group" - group_started=false - continue - fi - group_started=true + # Start server once for this TP×DP config. + if ! start_server "$tp" "$dp"; then + log "ERROR: ${config_label} failed to start; skipping all scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" 0 "SKIPPED_SERVICE_START_FAILED" "service failed to start" "$tp" "$dp" + log "===== ${config_label} DONE =====" + return 0 + fi - # Warmup with the shortest output for this ISL. - run_warmup "$tp" "$dp" "$isl" 128 + # Warmup with a small prompt before the first scenario. + run_warmup 1024 128 || true + + # Read scenarios into an array so we can skip remaining entries on failure. + local -a scenarios=() + while IFS= read -r line; do + scenarios+=("$line") + done < <(tail -n +2 "$scenario_tsv") + + local i mark isl dsl conc num + local output_file detail_log gpu_csv sname bench_rc + for (( i = 0; i < ${#scenarios[@]}; i++ )); do + IFS=$'\t' read -r mark isl dsl conc num <<< "${scenarios[$i]}" + + if [[ "$GRID_LIMIT" -gt 0 && "$i" -ge "$GRID_LIMIT" ]]; then + log "GRID_LIMIT=${GRID_LIMIT} reached; skipping remaining scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_GRID_LIMIT" "GRID_LIMIT reached" "$tp" "$dp" + break fi - # If the server for this group did not start, skip all scenarios in it. - if [[ "$group_started" != true ]]; then - log "skipping ${config_label} ISL=${isl} scenario (server not started)" - continue - fi + sname="c${conc}_i${isl}_o${dsl}" + output_file="${raw_dir}/vllm_main_${sname}.jsonl" + detail_log="${phase_log_dir}/vllm_${config_label}_${sname}.log" + gpu_csv="${gpu_log_dir}/gpu_mem_${sname}.csv" - local output_file="${raw_dir}/vllm_main_c${conc}_i${isl}_o${dsl}.jsonl" - local detail_log="${phase_log_dir}/vllm_${config_label}_c${conc}_i${isl}_o${dsl}.log" - local gpu_csv="${gpu_log_dir}/gpu_mem_c${conc}_i${isl}_o${dsl}.csv" - - if scenario_already_completed "$output_file" "$num"; then - log "skipping already-completed ${config_label} scenario: c=${conc} i=${isl} o=${dsl}" + if scenario_already_completed "$output_file" "$num" || scenario_already_processed "$result_root" "$sname"; then + log "skipping already-processed ${config_label} scenario: ${sname}" continue fi @@ -277,7 +372,7 @@ run_parallel_config() { local gpu_pid gpu_pid="$(start_gpu_monitor "$gpu_csv")" - local bench_rc=0 + bench_rc=0 timeout "$SCENARIO_TIMEOUT_S" \ "$PYTHON" -m sglang.bench_serving \ --backend vllm \ @@ -295,25 +390,56 @@ run_parallel_config() { stop_gpu_monitor "$gpu_pid" - if [[ "$bench_rc" -ne 0 ]]; then - log "ERROR: ${config_label} scenario c=${conc} i=${isl} o=${dsl} failed (rc=${bench_rc}); see ${detail_log}" - if [[ "$mark" == "P" ]]; then - log "optional (P) scenario failed; recording as skipped_oom and continuing" - append_scenario_record "$result_root" \ - "name=c${conc}_i${isl}_o${dsl}" \ - "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ - "status=\"skipped_oom\"" \ - "note=\"bench command failed or timed out (rc=${bench_rc})\"" - continue - else - log "mandatory (Y) scenario failed; aborting current ISL group" - stop_server "$tp" "$dp" - group_started=false - continue - fi + if [[ "$bench_rc" -eq 0 ]]; then + log "finished ${config_label} scenario: output=${output_file}" + continue fi - log "finished ${config_label} scenario: output=${output_file}" + # Failure handling. + if detect_oom "$detail_log" "${log_dir_global}/vllm_tp${tp}_dp${dp}.server.outer.log"; then + log "ERROR: ${config_label} scenario ${sname} triggered OOM; stopping config" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"OOM\"" \ + "note=\"detected CUDA out-of-memory\"" + record_skipped_csv "$skipped_csv" \ + "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=OOM" "reason=detected CUDA out-of-memory" "detail_log=${detail_log}" + stop_server "$tp" "$dp" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_AFTER_OOM" "previous case OOM" "$tp" "$dp" + break + fi + + log "ERROR: ${config_label} scenario ${sname} failed (rc=${bench_rc}); see ${detail_log}" + if [[ "$mark" == "P" ]]; then + log "optional (P) scenario failed; recording as skipped and continuing" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"skipped_optional\"" \ + "note=\"optional scenario failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=skipped_optional" "reason=optional scenario failed (rc=${bench_rc})" "detail_log=${detail_log}" + continue + fi + + # Mandatory scenario failed but not OOM: try to restart the server. + if restart_server "$tp" "$dp"; then + run_warmup 1024 128 || true + log "resuming ${config_label} after server restart" + continue + fi + + log "ERROR: ${config_label} server restart failed; skipping remaining scenarios" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"FAILED\"" \ + "note=\"scenario failed and server restart failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=FAILED" "reason=scenario failed and server restart failed" "detail_log=${detail_log}" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_RESTART_FAILED" "server restart failed" "$tp" "$dp" + break done stop_server "$tp" "$dp"