From e0ae0715b14a29965fb9f339db6cd453b9e55037 Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Tue, 14 Jul 2026 10:41:01 +0000 Subject: [PATCH] feat(adaptive): add RESUME_RUN_ID resume support and rename DSL to OSL - Add RESUME_RUN_ID env var for breakpoint resume in adaptive benchmarks. When set, the script reuses an existing result directory and skips already-tested (TP, DP, ISL, OSL) shapes based on adaptive_shapes.jsonl. - Fix RUN_ID unbound variable in resume mode. - Fix jq query to use -s (slurp) for jsonl files. - Add resume skip logic to DRY_RUN mode as well. - Rename DSL -> OSL across all adaptive benchmark files for consistency: - scripts/common/adaptive_bench_lib.sh - scripts/common/adaptive_concurrency.py - experiments/dsv4_h200_vllm_tp_dp_matrix/adaptive_config.env - experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh - experiments/dsv4_h200_sglang_tp_dp_matrix/adaptive_config.env - experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh - experiments/ADAPTIVE_CONCURRENCY_USAGE.md --- experiments/ADAPTIVE_CONCURRENCY_USAGE.md | 14 +- .../adaptive_config.env | 10 +- .../run_adaptive_concurrency.sh | 6 +- .../adaptive_config.env | 10 +- .../run_adaptive_concurrency.sh | 6 +- scripts/common/adaptive_bench_lib.sh | 124 ++++++++++++------ scripts/common/adaptive_concurrency.py | 22 ++-- 7 files changed, 121 insertions(+), 71 deletions(-) diff --git a/experiments/ADAPTIVE_CONCURRENCY_USAGE.md b/experiments/ADAPTIVE_CONCURRENCY_USAGE.md index 4e28428..3a1093b 100644 --- a/experiments/ADAPTIVE_CONCURRENCY_USAGE.md +++ b/experiments/ADAPTIVE_CONCURRENCY_USAGE.md @@ -1,6 +1,6 @@ # 自适应并发 Benchmark 使用说明 -这套脚本对每个固定的 `(TP, DP, ISL, DSL)` 依次测试 +这套脚本对每个固定的 `(TP, DP, ISL, OSL)` 依次测试 `C=1,2,4,8,...`，直到 Total TPS 连续两次增长不足 2%，或达到并发上限、 发生 OOM。每个正式测点发送 `C * 5` 个请求，并先用相同并发 C warm-up。 @@ -26,7 +26,7 @@ bash run_adaptive_concurrency.sh ## 2. 先 dry-run -只打印 TP/DP、ISL/DSL 和并发搜索计划，不加载模型： +只打印 TP/DP、ISL/OSL 和并发搜索计划，不加载模型： ```bash DRY_RUN=1 bash run_adaptive_concurrency.sh @@ -34,13 +34,13 @@ DRY_RUN=1 bash run_adaptive_concurrency.sh ## 3. 单组合 smoke -只测 `TP=8, DP=1, ISL=1K, DSL=128`，并发搜索到 8： +只测 `TP=8, DP=1, ISL=1K, OSL=128`，并发搜索到 8： ```bash RUN_ID=smoke-$(date +%Y%m%d-%H%M%S) \ TP_LIST="8" \ ISL_LIST="1024" \ -DSL_LIST="128" \ +OSL_LIST="128" \ GRID_LIMIT=1 \ SEARCH_MAX_CONCURRENCY=8 \ RESULT_BASE=/data3/yy/sskj_adaptive_results/smoke \ @@ -97,8 +97,8 @@ bash run_adaptive_concurrency.sh - vLLM 端口用 `VLLM_PORT`，SGLang 端口用 `SGLANG_PORT`。 - `TP_LIST="8"` 只跑 TP=8；DP 使用 `config.env` 中对应的 TP/DP 组合。 -- `ISL_LIST="1024 4096"`、`DSL_LIST="128 256"` 用来筛选长度。 -- `GRID_LIMIT=1` 表示每组 TP/DP 只跑一个 ISL/DSL shape。 +- `ISL_LIST="1024 4096"`、`OSL_LIST="128 256"` 用来筛选长度。 +- `GRID_LIMIT=1` 表示每组 TP/DP 只跑一个 ISL/OSL shape。 - `BENCH_WARMUP_MAX_REQUESTS=0` 表示每个测点用完整并发 C warm-up；正数表示上限。 - 正式基线保持 `BENCH_DATASET_NAME=random` 和 `RANDOM_RANGE_RATIO=1.0`。 - `random-ids` 只用于检查链路，不与正式 ShareGPT-backed 结果混用。 @@ -112,7 +112,7 @@ bash run_adaptive_concurrency.sh ```text adaptive_points.csv 每个并发测点的 TPS、TTFT、TPOT 等 -adaptive_summary.csv 每个 ISL/DSL/TP/DP 的饱和并发和最佳 TPS 并发 +adaptive_summary.csv 每个 ISL/OSL/TP/DP 的饱和并发和最佳 TPS 并发 adaptive_summary.md 可直接阅读的汇总表 run_manifest.json 本次模型、数据集和搜索参数 logs/ 编排与服务日志 diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/adaptive_config.env b/experiments/dsv4_h200_sglang_tp_dp_matrix/adaptive_config.env index 4085397..1eda1f0 100644 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/adaptive_config.env +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/adaptive_config.env @@ -1,6 +1,6 @@ # Adaptive concurrency search settings. # -# For each fixed (TP, DP, ISL, DSL), probe: +# For each fixed (TP, DP, ISL, OSL), probe: # C = start, start * multiplier, ... up to max # and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for # PLATEAU_PATIENCE consecutive points. @@ -21,7 +21,7 @@ PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" # random-ids to use generated token IDs without a ShareGPT seed dataset. BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" # SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed -# ISL/DSL points; lower values intentionally benchmark a length distribution. +# ISL/OSL points; lower values intentionally benchmark a length distribution. RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" # Before each measured point, warm up with the same concurrency so lazy kernel # compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no @@ -39,11 +39,11 @@ SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" # Optional space-separated filters, useful for smoke tests: -# TP_LIST="8" ISL_LIST="1024" DSL_LIST="128" +# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128" TP_LIST="${TP_LIST:-}" ISL_LIST="${ISL_LIST:-}" -DSL_LIST="${DSL_LIST:-}" +OSL_LIST="${OSL_LIST:-}" DRY_RUN="${DRY_RUN:-0}" -# Counts ISL/DSL shapes per TP/DP config, not individual concurrency probes. +# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes. GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh index 90f7827..0979a46 100755 --- a/experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/DSL shape. +# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape. set -Eeuo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -130,7 +130,7 @@ engine_detect_oom() { engine_run_bench() { local isl="$1" - local dsl="$2" + local osl="$2" local concurrency="$3" local num_prompts="$4" local output_file="$5" @@ -142,7 +142,7 @@ engine_run_bench() { --port "$ENGINE_PORT" --dataset-name "$BENCH_DATASET_NAME" --random-input-len "$isl" - --random-output-len "$dsl" + --random-output-len "$osl" --random-range-ratio "$RANDOM_RANGE_RATIO" --num-prompts "$num_prompts" --max-concurrency "$concurrency" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/adaptive_config.env b/experiments/dsv4_h200_vllm_tp_dp_matrix/adaptive_config.env index 4085397..1eda1f0 100644 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/adaptive_config.env +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/adaptive_config.env @@ -1,6 +1,6 @@ # Adaptive concurrency search settings. # -# For each fixed (TP, DP, ISL, DSL), probe: +# For each fixed (TP, DP, ISL, OSL), probe: # C = start, start * multiplier, ... up to max # and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for # PLATEAU_PATIENCE consecutive points. @@ -21,7 +21,7 @@ PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" # random-ids to use generated token IDs without a ShareGPT seed dataset. BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" # SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed -# ISL/DSL points; lower values intentionally benchmark a length distribution. +# ISL/OSL points; lower values intentionally benchmark a length distribution. RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" # Before each measured point, warm up with the same concurrency so lazy kernel # compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no @@ -39,11 +39,11 @@ SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" # Optional space-separated filters, useful for smoke tests: -# TP_LIST="8" ISL_LIST="1024" DSL_LIST="128" +# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128" TP_LIST="${TP_LIST:-}" ISL_LIST="${ISL_LIST:-}" -DSL_LIST="${DSL_LIST:-}" +OSL_LIST="${OSL_LIST:-}" DRY_RUN="${DRY_RUN:-0}" -# Counts ISL/DSL shapes per TP/DP config, not individual concurrency probes. +# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes. GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh index 8b1fdd0..ca23f6a 100755 --- a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/DSL shape. +# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/OSL shape. set -Eeuo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -111,7 +111,7 @@ engine_detect_oom() { engine_run_bench() { local isl="$1" - local dsl="$2" + local osl="$2" local concurrency="$3" local num_prompts="$4" local output_file="$5" @@ -123,7 +123,7 @@ engine_run_bench() { --port "$ENGINE_PORT" --dataset-name "$BENCH_DATASET_NAME" --random-input-len "$isl" - --random-output-len "$dsl" + --random-output-len "$osl" --random-range-ratio "$RANDOM_RANGE_RATIO" --num-prompts "$num_prompts" --max-concurrency "$concurrency" diff --git a/scripts/common/adaptive_bench_lib.sh b/scripts/common/adaptive_bench_lib.sh index c64f4f5..35dc5d2 100755 --- a/scripts/common/adaptive_bench_lib.sh +++ b/scripts/common/adaptive_bench_lib.sh @@ -115,7 +115,7 @@ adaptive_append_point_from_metrics() { local dp="$3" local mark="$4" local isl="$5" - local dsl="$6" + local osl="$6" local concurrency="$7" local num_prompts="$8" local attempt="$9" @@ -133,7 +133,7 @@ adaptive_append_point_from_metrics() { --argjson dp "$dp" \ --arg mark "$mark" \ --argjson isl "$isl" \ - --argjson dsl "$dsl" \ + --argjson osl "$osl" \ --argjson concurrency "$concurrency" \ --argjson num_prompts "$num_prompts" \ --argjson warmup_requests "$(adaptive_warmup_request_count "$concurrency")" \ @@ -149,7 +149,7 @@ adaptive_append_point_from_metrics() { dp: $dp, mark: $mark, isl: $isl, - dsl: $dsl, + osl: $osl, concurrency: $concurrency, num_prompts: $num_prompts, warmup_requests: $warmup_requests, @@ -167,7 +167,7 @@ adaptive_append_failed_point() { local dp="$2" local mark="$3" local isl="$4" - local dsl="$5" + local osl="$5" local concurrency="$6" local num_prompts="$7" local attempt="$8" @@ -183,7 +183,7 @@ adaptive_append_failed_point() { --argjson dp "$dp" \ --arg mark "$mark" \ --argjson isl "$isl" \ - --argjson dsl "$dsl" \ + --argjson osl "$osl" \ --argjson concurrency "$concurrency" \ --argjson num_prompts "$num_prompts" \ --argjson warmup_requests "$(adaptive_warmup_request_count "$concurrency")" \ @@ -199,7 +199,7 @@ adaptive_append_failed_point() { dp: $dp, mark: $mark, isl: $isl, - dsl: $dsl, + osl: $osl, concurrency: $concurrency, num_prompts: $num_prompts, warmup_requests: $warmup_requests, @@ -221,7 +221,7 @@ adaptive_append_shape_summary() { local dp="$2" local mark="$3" local isl="$4" - local dsl="$5" + local osl="$5" local status="$6" local stop_reason="$7" local tested_points="$8" @@ -245,7 +245,7 @@ adaptive_append_shape_summary() { --argjson dp "$dp" \ --arg mark "$mark" \ --argjson isl "$isl" \ - --argjson dsl "$dsl" \ + --argjson osl "$osl" \ --arg status "$status" \ --arg stop_reason "$stop_reason" \ --argjson tested_points "$tested_points" \ @@ -263,7 +263,7 @@ adaptive_append_shape_summary() { dp: $dp, mark: $mark, isl: $isl, - dsl: $dsl, + osl: $osl, status: $status, stop_reason: $stop_reason, tested_points: $tested_points, @@ -281,7 +281,7 @@ adaptive_run_point() { local tp="$1" local dp="$2" local isl="$3" - local dsl="$4" + local osl="$4" local concurrency="$5" local num_prompts="$6" local config_root="$7" @@ -295,17 +295,17 @@ adaptive_run_point() { for (( attempt = 1; attempt <= MAX_POINT_RETRIES + 1; attempt++ )); do POINT_ATTEMPT="$attempt" - POINT_RAW_FILE="${config_root}/raw_outputs/${ENGINE}_adaptive_c${concurrency}_i${isl}_o${dsl}_a${attempt}.jsonl" - POINT_DETAIL_LOG="${config_root}/logs/${ENGINE}_c${concurrency}_i${isl}_o${dsl}_a${attempt}.log" - POINT_METRICS_FILE="${config_root}/metrics/${ENGINE}_c${concurrency}_i${isl}_o${dsl}_a${attempt}.json" - local gpu_csv="${config_root}/gpu_logs/gpu_c${concurrency}_i${isl}_o${dsl}_a${attempt}.csv" + POINT_RAW_FILE="${config_root}/raw_outputs/${ENGINE}_adaptive_c${concurrency}_i${isl}_o${osl}_a${attempt}.jsonl" + POINT_DETAIL_LOG="${config_root}/logs/${ENGINE}_c${concurrency}_i${isl}_o${osl}_a${attempt}.log" + POINT_METRICS_FILE="${config_root}/metrics/${ENGINE}_c${concurrency}_i${isl}_o${osl}_a${attempt}.json" + local gpu_csv="${config_root}/gpu_logs/gpu_c${concurrency}_i${isl}_o${osl}_a${attempt}.csv" - log "probe ${ENGINE} tp=${tp} dp=${dp} isl=${isl} dsl=${dsl} c=${concurrency} warmup=$(adaptive_warmup_request_count "$concurrency") n=${num_prompts} attempt=${attempt}" + log "probe ${ENGINE} tp=${tp} dp=${dp} isl=${isl} osl=${osl} c=${concurrency} warmup=$(adaptive_warmup_request_count "$concurrency") n=${num_prompts} attempt=${attempt}" gpu_pid="$(adaptive_start_gpu_monitor "$gpu_csv")" bench_rc=0 timeout "$SCENARIO_TIMEOUT_S" bash -c \ 'engine_run_bench "$@"' _ \ - "$isl" "$dsl" "$concurrency" "$num_prompts" "$POINT_RAW_FILE" \ + "$isl" "$osl" "$concurrency" "$num_prompts" "$POINT_RAW_FILE" \ > "$POINT_DETAIL_LOG" 2>&1 || bench_rc=$? adaptive_stop_gpu_monitor "$gpu_pid" @@ -316,7 +316,7 @@ adaptive_run_point() { --output "$POINT_METRICS_FILE" \ --expected-prompts "$num_prompts" \ --isl "$isl" \ - --dsl "$dsl" \ + --osl "$osl" \ --input-tolerance-pct "$INPUT_LENGTH_TOLERANCE_PCT" \ --output-tolerance-pct "$OUTPUT_LENGTH_TOLERANCE_PCT" \ >> "$POINT_DETAIL_LOG" 2>&1 || parse_rc=$? @@ -350,7 +350,7 @@ adaptive_run_shape() { local dp="$2" local mark="$3" local isl="$4" - local dsl="$5" + local osl="$5" local config_root="$6" local concurrency="$SEARCH_START_CONCURRENCY" local previous_tps="" @@ -376,7 +376,7 @@ adaptive_run_shape() { while (( concurrency <= SEARCH_MAX_CONCURRENCY )); do local num_prompts=$((concurrency * NUM_PROMPTS_MULTIPLIER)) point_rc=0 - adaptive_run_point "$tp" "$dp" "$isl" "$dsl" "$concurrency" "$num_prompts" "$config_root" || point_rc=$? + adaptive_run_point "$tp" "$dp" "$isl" "$osl" "$concurrency" "$num_prompts" "$config_root" || point_rc=$? tested_points=$((tested_points + 1)) stop_probe_concurrency="$concurrency" @@ -409,7 +409,7 @@ adaptive_run_shape() { max_successful_concurrency="$concurrency" last_total_tps="$current_tps" adaptive_append_point_from_metrics \ - "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$dsl" \ + "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \ "$concurrency" "$num_prompts" "$POINT_ATTEMPT" "$gain_pct" \ "$plateau_streak" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" @@ -422,7 +422,7 @@ adaptive_run_shape() { previous_tps="$current_tps" elif (( point_rc == 20 )); then adaptive_append_failed_point \ - "$tp" "$dp" "$mark" "$isl" "$dsl" "$concurrency" "$num_prompts" \ + "$tp" "$dp" "$mark" "$isl" "$osl" "$concurrency" "$num_prompts" \ "$POINT_ATTEMPT" "OOM" "$POINT_ERROR_TYPE" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" shape_status="OOM_BOUNDARY" stop_reason="OOM" @@ -430,7 +430,7 @@ adaptive_run_shape() { break elif (( point_rc == 21 )); then adaptive_append_point_from_metrics \ - "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$dsl" \ + "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \ "$concurrency" "$num_prompts" "$POINT_ATTEMPT" "" 0 \ "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" shape_status="INVALID_WORKLOAD" @@ -439,7 +439,7 @@ adaptive_run_shape() { break else adaptive_append_failed_point \ - "$tp" "$dp" "$mark" "$isl" "$dsl" "$concurrency" "$num_prompts" \ + "$tp" "$dp" "$mark" "$isl" "$osl" "$concurrency" "$num_prompts" \ "$POINT_ATTEMPT" "FAILED" "$POINT_ERROR_TYPE" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" shape_status="FAILED" stop_reason="BENCHMARK_FAILED" @@ -464,11 +464,11 @@ adaptive_run_shape() { done adaptive_append_shape_summary \ - "$tp" "$dp" "$mark" "$isl" "$dsl" "$shape_status" "$stop_reason" \ + "$tp" "$dp" "$mark" "$isl" "$osl" "$shape_status" "$stop_reason" \ "$tested_points" "$max_successful_concurrency" "$saturation_concurrency" \ "$stop_probe_concurrency" "$best_concurrency" "$best_tps" "$last_total_tps" - log "shape done tp=${tp} dp=${dp} isl=${isl} dsl=${dsl} stop=${stop_reason} saturation_c=${saturation_concurrency:-none} best_c=${best_concurrency:-none} best_total_tps=${best_tps:-none}" + log "shape done tp=${tp} dp=${dp} isl=${isl} osl=${osl} stop=${stop_reason} saturation_c=${saturation_concurrency:-none} best_c=${best_concurrency:-none} best_total_tps=${best_tps:-none}" } adaptive_matches_filter() { @@ -507,19 +507,61 @@ adaptive_cleanup_active_server() { fi } +adaptive_shape_already_tested() { + local tp="$1" + local dp="$2" + local mark="$3" + local isl="$4" + local osl="$5" + local resume_shapes_jsonl="$6" + if [[ ! -f "$resume_shapes_jsonl" ]]; then + return 1 + fi + local count + count="$(jq -s -r --argjson tp "$tp" --argjson dp "$dp" --arg mark "$mark" --argjson isl "$isl" --argjson osl "$osl" ' + [ .[] | select(.tp == $tp and .dp == $dp and .mark == $mark and .isl == $isl and .osl == $osl) ] | length + ' "$resume_shapes_jsonl" 2>/dev/null)" + if [[ -z "$count" ]]; then + return 1 + fi + if (( count > 0 )); then + return 0 + fi + return 1 +} + adaptive_main() { - RUN_ID="${RUN_ID:-adaptive_$(date '+%Y%m%d-%H%M%S')}" + local resume_mode=0 + local resume_shapes_jsonl="" + if [[ -n "${RESUME_RUN_ID:-}" ]]; then + ADAPTIVE_RUN_ROOT="${RESULT_BASE}/${RESUME_RUN_ID}" + if [[ ! -d "$ADAPTIVE_RUN_ROOT" ]]; then + log "ERROR: RESUME_RUN_ID=${RESUME_RUN_ID} does not exist under ${RESULT_BASE}" + return 1 + fi + resume_mode=1 + RUN_ID="$RESUME_RUN_ID" + resume_shapes_jsonl="${ADAPTIVE_RUN_ROOT}/adaptive_shapes.jsonl" + if [[ ! -f "$resume_shapes_jsonl" ]]; then + log "WARNING: no adaptive_shapes.jsonl found in ${ADAPTIVE_RUN_ROOT}; will resume with empty history" + fi + log "RESUME mode: continuing run_id=${RESUME_RUN_ID}" + else + RUN_ID="${RUN_ID:-adaptive_$(date '+%Y%m%d-%H%M%S')}" + ADAPTIVE_RUN_ROOT="${RESULT_BASE}/${RUN_ID}" + fi RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" MATRIX_MODE="${MATRIX_MODE:-Y}" - ADAPTIVE_RUN_ROOT="${RESULT_BASE}/${RUN_ID}" ADAPTIVE_LOG_DIR="${ADAPTIVE_RUN_ROOT}/logs" ADAPTIVE_POINTS_JSONL="${ADAPTIVE_RUN_ROOT}/adaptive_points.jsonl" ADAPTIVE_SHAPES_JSONL="${ADAPTIVE_RUN_ROOT}/adaptive_shapes.jsonl" local shapes_tsv="${ADAPTIVE_RUN_ROOT}/shapes.tsv" mkdir -p "$ADAPTIVE_LOG_DIR" - : > "$ADAPTIVE_POINTS_JSONL" - : > "$ADAPTIVE_SHAPES_JSONL" + if (( resume_mode == 0 )); then + : > "$ADAPTIVE_POINTS_JSONL" + : > "$ADAPTIVE_SHAPES_JSONL" + fi log_init "${ADAPTIVE_LOG_DIR}/orchestrator.log" adaptive_validate_config @@ -611,7 +653,7 @@ adaptive_main() { trap 'adaptive_cleanup_active_server; exit 143' TERM local abort_all=0 - local cfg tp dp config_root server_args shape_count mark isl dsl + local cfg tp dp config_root server_args shape_count mark isl osl for cfg in "${PARALLEL_CONFIGS[@]}"; do read -r tp dp <<< "$cfg" if ! adaptive_matches_filter "$tp" "$TP_LIST"; then @@ -626,15 +668,19 @@ adaptive_main() { if [[ "$DRY_RUN" == "1" ]]; then log "DRY_RUN server tp=${tp} dp=${dp}: ${server_args}" shape_count=0 - while IFS=$'\t' read -r mark isl dsl; do + while IFS=$'\t' read -r mark isl osl; do [[ "$mark" == "mark" ]] && continue adaptive_matches_filter "$isl" "$ISL_LIST" || continue - adaptive_matches_filter "$dsl" "$DSL_LIST" || continue + adaptive_matches_filter "$osl" "$OSL_LIST" || continue if (( GRID_LIMIT > 0 && shape_count >= GRID_LIMIT )); then break fi shape_count=$((shape_count + 1)) - log "DRY_RUN tp=${tp} dp=${dp} isl=${isl} dsl=${dsl} C=[$(adaptive_print_sequence)]" + if (( resume_mode == 1 )) && adaptive_shape_already_tested "$tp" "$dp" "$mark" "$isl" "$osl" "$resume_shapes_jsonl"; then + log "DRY_RUN RESUME skip already tested shape tp=${tp} dp=${dp} isl=${isl} osl=${osl} mark=${mark}" + continue + fi + log "DRY_RUN tp=${tp} dp=${dp} isl=${isl} osl=${osl} C=[$(adaptive_print_sequence)]" done < "$shapes_tsv" continue fi @@ -653,17 +699,21 @@ adaptive_main() { } shape_count=0 - while IFS=$'\t' read -r mark isl dsl; do + while IFS=$'\t' read -r mark isl osl; do [[ "$mark" == "mark" ]] && continue adaptive_matches_filter "$isl" "$ISL_LIST" || continue - adaptive_matches_filter "$dsl" "$DSL_LIST" || continue + adaptive_matches_filter "$osl" "$OSL_LIST" || continue if (( GRID_LIMIT > 0 && shape_count >= GRID_LIMIT )); then log "GRID_LIMIT=${GRID_LIMIT} reached for tp=${tp} dp=${dp}" break fi shape_count=$((shape_count + 1)) - log "===== shape START tp=${tp} dp=${dp} isl=${isl} dsl=${dsl} mark=${mark} =====" - adaptive_run_shape "$tp" "$dp" "$mark" "$isl" "$dsl" "$config_root" + if (( resume_mode == 1 )) && adaptive_shape_already_tested "$tp" "$dp" "$mark" "$isl" "$osl" "$resume_shapes_jsonl"; then + log "RESUME skip already tested shape tp=${tp} dp=${dp} isl=${isl} osl=${osl} mark=${mark}" + continue + fi + log "===== shape START tp=${tp} dp=${dp} isl=${isl} osl=${osl} mark=${mark} =====" + adaptive_run_shape "$tp" "$dp" "$mark" "$isl" "$osl" "$config_root" if (( ADAPTIVE_FATAL_WORKLOAD == 1 )); then log "ERROR: invalid synthetic workload; aborting remaining searches" diff --git a/scripts/common/adaptive_concurrency.py b/scripts/common/adaptive_concurrency.py index d7a517b..e8de21a 100755 --- a/scripts/common/adaptive_concurrency.py +++ b/scripts/common/adaptive_concurrency.py @@ -17,7 +17,7 @@ POINT_FIELDS = [ "dp", "mark", "isl", - "dsl", + "osl", "concurrency", "num_prompts", "warmup_requests", @@ -60,7 +60,7 @@ SHAPE_FIELDS = [ "dp", "mark", "isl", - "dsl", + "osl", "status", "stop_reason", "tested_points", @@ -114,16 +114,16 @@ def command_shapes(args: argparse.Namespace) -> int: mode = args.mode or data.get("mode", "Y") print("mark\tinput_len\toutput_len") for isl_text in sorted(data["matrix"], key=int): - dsl_map = data["matrix"][isl_text] - for dsl_text in sorted(dsl_map, key=int): - mark = dsl_map[dsl_text] + osl_map = data["matrix"][isl_text] + for osl_text in sorted(osl_map, key=int): + mark = osl_map[osl_text] if mode == "Y" and mark != "Y": continue if mode == "Y+P" and mark not in ("Y", "P"): continue if mode != "all" and mark == "N": continue - print(f"{mark}\t{int(isl_text)}\t{int(dsl_text)}") + print(f"{mark}\t{int(isl_text)}\t{int(osl_text)}") return 0 @@ -175,8 +175,8 @@ def command_parse_result(args: argparse.Namespace) -> int: f"mean_input_tokens={mean_input:.2f}, expected_range={input_low:.2f}-{input_high:.2f}" ) - output_low = args.dsl * (1.0 - args.output_tolerance_pct / 100.0) - output_high = args.dsl * (1.0 + args.output_tolerance_pct / 100.0) + output_low = args.osl * (1.0 - args.output_tolerance_pct / 100.0) + output_high = args.osl * (1.0 + args.output_tolerance_pct / 100.0) if not output_low <= mean_output <= output_high: validation_errors.append( f"mean_output_tokens={mean_output:.2f}, expected_range={output_low:.2f}-{output_high:.2f}" @@ -256,12 +256,12 @@ def command_summarize(args: argparse.Namespace) -> int: with report.open("w", encoding="utf-8") as f: f.write("# Adaptive concurrency search summary\n\n") f.write( - "| Engine | TP | DP | ISL | DSL | Stop | Saturation C | Best TPS C | Best Total TPS | Max successful C |\n" + "| Engine | TP | DP | ISL | OSL | Stop | Saturation C | Best TPS C | Best Total TPS | Max successful C |\n" ) f.write("|---|---:|---:|---:|---:|---|---:|---:|---:|---:|\n") for row in shapes: f.write( - "| {engine} | {tp} | {dp} | {isl} | {dsl} | {stop_reason} | " + "| {engine} | {tp} | {dp} | {isl} | {osl} | {stop_reason} | " "{saturation_concurrency} | {best_tps_concurrency} | {best_total_tps} | " "{max_successful_concurrency} |\n".format(**{k: row.get(k, "") for k in SHAPE_FIELDS}) ) @@ -292,7 +292,7 @@ def build_parser() -> argparse.ArgumentParser: parse_result.add_argument("--output", type=Path, required=True) parse_result.add_argument("--expected-prompts", type=int, required=True) parse_result.add_argument("--isl", type=int, required=True) - parse_result.add_argument("--dsl", type=int, required=True) + parse_result.add_argument("--osl", type=int, required=True) parse_result.add_argument("--input-tolerance-pct", type=float, default=5.0) parse_result.add_argument("--output-tolerance-pct", type=float, default=10.0) parse_result.set_defaults(func=command_parse_result)