e0ae0715b1
- Add RESUME_RUN_ID env var for breakpoint resume in adaptive benchmarks. When set, the script reuses an existing result directory and skips already-tested (TP, DP, ISL, OSL) shapes based on adaptive_shapes.jsonl. - Fix RUN_ID unbound variable in resume mode. - Fix jq query to use -s (slurp) for jsonl files. - Add resume skip logic to DRY_RUN mode as well. - Rename DSL -> OSL across all adaptive benchmark files for consistency: - scripts/common/adaptive_bench_lib.sh - scripts/common/adaptive_concurrency.py - experiments/dsv4_h200_vllm_tp_dp_matrix/adaptive_config.env - experiments/dsv4_h200_vllm_tp_dp_matrix/run_adaptive_concurrency.sh - experiments/dsv4_h200_sglang_tp_dp_matrix/adaptive_config.env - experiments/dsv4_h200_sglang_tp_dp_matrix/run_adaptive_concurrency.sh - experiments/ADAPTIVE_CONCURRENCY_USAGE.md
50 lines
2.3 KiB
Bash
50 lines
2.3 KiB
Bash
# Adaptive concurrency search settings.
|
|
#
|
|
# For each fixed (TP, DP, ISL, OSL), probe:
|
|
# C = start, start * multiplier, ... up to max
|
|
# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for
|
|
# PLATEAU_PATIENCE consecutive points.
|
|
|
|
SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
|
|
SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}"
|
|
SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}"
|
|
NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"
|
|
|
|
# A gain below 2% is treated as throughput saturation. Two consecutive
|
|
# low-gain points prevent one noisy measurement from stopping the search.
|
|
TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}"
|
|
PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}"
|
|
|
|
# Keep the same random workload semantics as the fixed matrix baseline.
|
|
# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times
|
|
# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to
|
|
# random-ids to use generated token IDs without a ShareGPT seed dataset.
|
|
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
|
|
# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed
|
|
# ISL/OSL points; lower values intentionally benchmark a length distribution.
|
|
RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}"
|
|
# Before each measured point, warm up with the same concurrency so lazy kernel
|
|
# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no
|
|
# cap; set a positive cap only when very high-concurrency warmup is impractical.
|
|
BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}"
|
|
|
|
# Reject a point if the completed request count or actual token lengths do not
|
|
# match the requested workload.
|
|
INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}"
|
|
OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}"
|
|
|
|
MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}"
|
|
SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}"
|
|
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
|
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
|
|
|
# Optional space-separated filters, useful for smoke tests:
|
|
# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128"
|
|
TP_LIST="${TP_LIST:-}"
|
|
ISL_LIST="${ISL_LIST:-}"
|
|
OSL_LIST="${OSL_LIST:-}"
|
|
|
|
DRY_RUN="${DRY_RUN:-0}"
|
|
# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes.
|
|
GRID_LIMIT="${GRID_LIMIT:-0}"
|