dsv4_h200_vllm_tp_dp_matrix: single server per config, OOM/restart handling, DRY_RUN/GRID_LIMIT, include 1M

experiments/dsv4_h200_vllm_tp_dp_matrix/config.env

@@ -39,3 +39,10 @@ SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
 
 # GPU memory sampling interval (seconds).
 GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
+
+# Dry-run mode: if 1, only log the server args and scenario plan without starting
+# any server or sending requests.
+DRY_RUN="${DRY_RUN:-0}"
+
+# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios.
+GRID_LIMIT="${GRID_LIMIT:-0}"

experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json

@@ -60,8 +60,8 @@
       "4096": "N"
     },
     "1048576": {
-      "128": "N",
+      "128": "Y",
-      "256": "N",
+      "256": "P",
       "512": "N",
       "1024": "N",
       "2048": "N",

experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh

@@ -18,6 +18,8 @@ MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}"
 MATRIX_MODE="${MATRIX_MODE:-Y}"
 SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
 GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
+DRY_RUN="${DRY_RUN:-0}"
+GRID_LIMIT="${GRID_LIMIT:-0}"
 
 PYTHON="${VENV_CLIENT}/bin/python"
 
@@ -26,7 +28,7 @@ mkdir -p "$log_dir_global"
 log_init "${log_dir_global}/orchestrator.log"
 
 log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}"
-log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE}"
+log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE} dry_run=${DRY_RUN} grid_limit=${GRID_LIMIT}"
 
 # ---------------------------------------------------------------------------
 # Helpers
@@ -95,13 +97,20 @@ start_server() {
   log "vllm server tp=${tp} dp=${dp} is healthy on port ${VLLM_PORT}"
 }
 
-run_warmup() {
+restart_server() {
   local tp="$1"
   local dp="$2"
-  local input_len="$3"
+  log "restarting vllm server tp=${tp} dp=${dp} after non-OOM failure"
-  local output_len="$4"
+  stop_server "$tp" "$dp"
+  sleep 10
+  start_server "$tp" "$dp"
+}
 
-  log "warming up tp=${tp} dp=${dp} (input=${input_len}, output=${output_len}, num=1)"
+run_warmup() {
+  local input_len="$1"
+  local output_len="$2"
+
+  log "warming up (input=${input_len}, output=${output_len}, num=1)"
   "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \
     --backend vllm \
     --host 127.0.0.1 \
@@ -110,7 +119,7 @@ run_warmup() {
     --output-len "$output_len" \
     --num 1 \
     --env-python "$PYTHON" \
-    >> "${log_dir_global}/vllm_tp${tp}_dp${dp}.warmup.log" 2>&1
+    >> "${log_dir_global}/warmup.log" 2>&1
   log "warmup completed"
 }
 
@@ -136,6 +145,37 @@ except Exception:
   [[ "${completed:-0}" -ge "$expected" ]]
 }
 
+scenario_already_processed() {
+  local result_root="$1"
+  local scenario_name="$2"
+  local json_path="${result_root}/results.json"
+  [[ -f "$json_path" ]] || return 1
+  "$PYTHON" -c "
+import json, sys
+path, name = sys.argv[1], sys.argv[2]
+try:
+    with open(path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+    for s in data.get('scenarios', []):
+        if s.get('name') == name:
+            if s.get('status') or s.get('metrics', {}).get('success', 0) > 0:
+                sys.exit(0)
+except Exception:
+    pass
+sys.exit(1)
+" "$json_path" "$scenario_name"
+}
+
+detect_oom() {
+  local detail_log="$1"
+  local server_outer_log="$2"
+  local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
+  if grep -Eiq "$pattern" "$detail_log" "$server_outer_log" 2>/dev/null; then
+    return 0
+  fi
+  return 1
+}
+
 start_gpu_monitor() {
   local csv_path="$1"
   mkdir -p "$(dirname "$csv_path")"
@@ -160,7 +200,6 @@ append_scenario_record() {
   local result_root="$1"
   local json_path="$result_root/results.json"
   shift
-  # Remaining args are key=value pairs.
   local scenario_json
   scenario_json="$("$PYTHON" -c "
 import json, sys
@@ -171,6 +210,54 @@ print(json.dumps(d, ensure_ascii=False))
   PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json"
 }
 
+record_skipped_csv() {
+  local csv_path="$1"
+  shift
+  # Args: key=value
+  local row
+  row="$("$PYTHON" -c "
+import csv, json, sys, io
+pairs = [a.split('=', 1) for a in sys.argv[1:]]
+d = {k: json.loads(v) for k, v in pairs}
+buf = io.StringIO()
+writer = csv.DictWriter(buf, fieldnames=['engine','tp','dp','mark','isl','dsl','concurrency','status','reason','detail_log'], extrasaction='ignore')
+writer.writerow(d)
+print(buf.getvalue().strip())
+" "$@")"
+  echo "$row" >> "$csv_path"
+}
+
+skip_remaining_scenarios() {
+  local result_root="$1"
+  local scenario_tsv="$2"
+  local start_index="$3"
+  local status="$4"
+  local reason="$5"
+  local tp="$6"
+  local dp="$7"
+  local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv"
+
+  local i=0
+  tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do
+    if (( i < start_index )); then
+      i=$((i + 1))
+      continue
+    fi
+    i=$((i + 1))
+    local sname="c${conc}_i${isl}_o${dsl}"
+    if scenario_already_processed "$result_root" "$sname"; then
+      continue
+    fi
+    append_scenario_record "$result_root" \
+      "name=${sname}" \
+      "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
+      "status=\"${status}\"" \
+      "note=\"${reason}\""
+    record_skipped_csv "$skipped_csv" \
+      "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=${status}" "reason=${reason}"
+  done
+}
+
 # ---------------------------------------------------------------------------
 # Per-configuration runner
 # ---------------------------------------------------------------------------
@@ -197,11 +284,6 @@ run_parallel_config() {
   total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)"
   log "generated ${total_scenarios} scenarios for ${config_label}"
 
-  if [[ "$total_scenarios" -eq 0 ]]; then
-    log "no scenarios for ${config_label}; skipping"
-    return 0
-  fi
-
   # Write metadata.
   ensure_result_root "$result_root"
   write_metadata_json \
@@ -218,7 +300,6 @@ run_parallel_config() {
     "$VENV_VLLM" \
     "H200 vLLM TP×DP matrix for DeepSeek-V4-Flash"
 
-  # Embed static config and the exact server command line used.
   local server_args_str
   server_args_str="$(build_server_args "$tp" "$dp")"
   jq --arg tp "$tp" --arg dp "$dp" --arg cuda "$CUDA_VISIBLE_DEVICES" --arg args "$server_args_str" \
@@ -232,43 +313,57 @@ run_parallel_config() {
      }' "${result_root}/results.json" > "${result_root}/results.json.tmp" && \
     mv "${result_root}/results.json.tmp" "${result_root}/results.json"
 
-  # Group scenarios by input_len.  We restart the server for each ISL to keep
+  if [[ "$DRY_RUN" == "1" ]]; then
-  # memory state isolated and to warm up per context length.
+    log "DRY_RUN: would start server with args: ${server_args_str}"
-  local current_isl=""
+    local line
-  local group_started=false
-
     tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do
-    # When the input length changes, restart the server for the new group.
+      log "DRY_RUN: ${config_label} scenario mark=${mark} c=${conc} i=${isl} o=${dsl} n=${num}"
-    if [[ "$isl" != "$current_isl" ]]; then
+    done
-      if [[ "$group_started" == true ]]; then
+    log "===== ${config_label} DONE (dry run) ====="
-        stop_server "$tp" "$dp"
+    return 0
   fi
-      current_isl="$isl"
 
-      stop_server "$tp" "$dp"
+  # Initialize skipped_after_oom.csv for this run.
+  local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv"
+  if [[ ! -f "$skipped_csv" ]]; then
+    echo "engine,tp,dp,mark,isl,dsl,concurrency,status,reason,detail_log" > "$skipped_csv"
+  fi
+
+  # Start server once for this TP×DP config.
   if ! start_server "$tp" "$dp"; then
-        log "ERROR: ${config_label} failed to start for ISL=${isl}; skipping this group"
+    log "ERROR: ${config_label} failed to start; skipping all scenarios"
-        group_started=false
+    skip_remaining_scenarios "$result_root" "$scenario_tsv" 0 "SKIPPED_SERVICE_START_FAILED" "service failed to start" "$tp" "$dp"
-        continue
+    log "===== ${config_label} DONE ====="
-      fi
+    return 0
-      group_started=true
-
-      # Warmup with the shortest output for this ISL.
-      run_warmup "$tp" "$dp" "$isl" 128
   fi
 
-    # If the server for this group did not start, skip all scenarios in it.
+  # Warmup with a small prompt before the first scenario.
-    if [[ "$group_started" != true ]]; then
+  run_warmup 1024 128 || true
-      log "skipping ${config_label} ISL=${isl} scenario (server not started)"
+
-      continue
+  # Read scenarios into an array so we can skip remaining entries on failure.
+  local -a scenarios=()
+  while IFS= read -r line; do
+    scenarios+=("$line")
+  done < <(tail -n +2 "$scenario_tsv")
+
+  local i mark isl dsl conc num
+  local output_file detail_log gpu_csv sname bench_rc
+  for (( i = 0; i < ${#scenarios[@]}; i++ )); do
+    IFS=$'\t' read -r mark isl dsl conc num <<< "${scenarios[$i]}"
+
+    if [[ "$GRID_LIMIT" -gt 0 && "$i" -ge "$GRID_LIMIT" ]]; then
+      log "GRID_LIMIT=${GRID_LIMIT} reached; skipping remaining scenarios"
+      skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_GRID_LIMIT" "GRID_LIMIT reached" "$tp" "$dp"
+      break
     fi
 
-    local output_file="${raw_dir}/vllm_main_c${conc}_i${isl}_o${dsl}.jsonl"
+    sname="c${conc}_i${isl}_o${dsl}"
-    local detail_log="${phase_log_dir}/vllm_${config_label}_c${conc}_i${isl}_o${dsl}.log"
+    output_file="${raw_dir}/vllm_main_${sname}.jsonl"
-    local gpu_csv="${gpu_log_dir}/gpu_mem_c${conc}_i${isl}_o${dsl}.csv"
+    detail_log="${phase_log_dir}/vllm_${config_label}_${sname}.log"
+    gpu_csv="${gpu_log_dir}/gpu_mem_${sname}.csv"
 
-    if scenario_already_completed "$output_file" "$num"; then
+    if scenario_already_completed "$output_file" "$num" || scenario_already_processed "$result_root" "$sname"; then
-      log "skipping already-completed ${config_label} scenario: c=${conc} i=${isl} o=${dsl}"
+      log "skipping already-processed ${config_label} scenario: ${sname}"
       continue
     fi
 
@@ -277,7 +372,7 @@ run_parallel_config() {
     local gpu_pid
     gpu_pid="$(start_gpu_monitor "$gpu_csv")"
 
-    local bench_rc=0
+    bench_rc=0
     timeout "$SCENARIO_TIMEOUT_S" \
       "$PYTHON" -m sglang.bench_serving \
         --backend vllm \
@@ -295,25 +390,56 @@ run_parallel_config() {
 
     stop_gpu_monitor "$gpu_pid"
 
-    if [[ "$bench_rc" -ne 0 ]]; then
+    if [[ "$bench_rc" -eq 0 ]]; then
-      log "ERROR: ${config_label} scenario c=${conc} i=${isl} o=${dsl} failed (rc=${bench_rc}); see ${detail_log}"
+      log "finished ${config_label} scenario: output=${output_file}"
-      if [[ "$mark" == "P" ]]; then
-        log "optional (P) scenario failed; recording as skipped_oom and continuing"
-        append_scenario_record "$result_root" \
-          "name=c${conc}_i${isl}_o${dsl}" \
-          "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
-          "status=\"skipped_oom\"" \
-          "note=\"bench command failed or timed out (rc=${bench_rc})\""
       continue
-      else
-        log "mandatory (Y) scenario failed; aborting current ISL group"
-        stop_server "$tp" "$dp"
-        group_started=false
-        continue
-      fi
     fi
 
-    log "finished ${config_label} scenario: output=${output_file}"
+    # Failure handling.
+    if detect_oom "$detail_log" "${log_dir_global}/vllm_tp${tp}_dp${dp}.server.outer.log"; then
+      log "ERROR: ${config_label} scenario ${sname} triggered OOM; stopping config"
+      append_scenario_record "$result_root" \
+        "name=${sname}" \
+        "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
+        "status=\"OOM\"" \
+        "note=\"detected CUDA out-of-memory\""
+      record_skipped_csv "$skipped_csv" \
+        "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=OOM" "reason=detected CUDA out-of-memory" "detail_log=${detail_log}"
+      stop_server "$tp" "$dp"
+      skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_AFTER_OOM" "previous case OOM" "$tp" "$dp"
+      break
+    fi
+
+    log "ERROR: ${config_label} scenario ${sname} failed (rc=${bench_rc}); see ${detail_log}"
+    if [[ "$mark" == "P" ]]; then
+      log "optional (P) scenario failed; recording as skipped and continuing"
+      append_scenario_record "$result_root" \
+        "name=${sname}" \
+        "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
+        "status=\"skipped_optional\"" \
+        "note=\"optional scenario failed (rc=${bench_rc})\""
+      record_skipped_csv "$skipped_csv" \
+        "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=skipped_optional" "reason=optional scenario failed (rc=${bench_rc})" "detail_log=${detail_log}"
+      continue
+    fi
+
+    # Mandatory scenario failed but not OOM: try to restart the server.
+    if restart_server "$tp" "$dp"; then
+      run_warmup 1024 128 || true
+      log "resuming ${config_label} after server restart"
+      continue
+    fi
+
+    log "ERROR: ${config_label} server restart failed; skipping remaining scenarios"
+    append_scenario_record "$result_root" \
+      "name=${sname}" \
+      "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
+      "status=\"FAILED\"" \
+      "note=\"scenario failed and server restart failed (rc=${bench_rc})\""
+    record_skipped_csv "$skipped_csv" \
+      "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=FAILED" "reason=scenario failed and server restart failed" "detail_log=${detail_log}"
+    skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_RESTART_FAILED" "server restart failed" "$tp" "$dp"
+    break
   done
 
   stop_server "$tp" "$dp"