# TP×DP matrix experiment for DeepSeek-V4-Flash-DSpark on H20 (8 GPUs) using vLLM.
# Tests vLLM with three parallel configurations:
#   TP=2, DP=4  -> 2 GPUs per replica, 4 replicas
#   TP=4, DP=2  -> 4 GPUs per replica, 2 replicas
#   TP=8, DP=1  -> 8 GPUs, no data parallelism

EXPERIMENT="dsv4_dspark_h20_vllm_tp_dp_matrix"
MODEL_NAME="DeepSeek-V4-Flash-DSpark"
MODEL_PATH="/data1/hf_models/DeepSeek-V4-Flash-DSpark"
SERVED_MODEL_NAME="deepseek-v4-flash-dspark"

VLLM_PORT="${VLLM_PORT:-30032}"

# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.)
# and the benchmark client. Defaults to the system python3 if the sglang venv
# does not exist on the host.
VENV_CLIENT="${VENV_CLIENT:-/data1/yy/sskj/envs/sglang}"

# Run the benchmark client natively (0) or inside Docker (1).
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"

export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"

# Runtime working directory for logs, pid files, and tmp. Defaults to a local
# directory under this experiment so the benchmark is self-contained.
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"

# Parallel configurations to test. Format: "TP DP"
declare -a PARALLEL_CONFIGS=(
  "2 4"
  "4 2"
  "8 1"
)

# vLLM server settings
MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}"
MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}"
KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}"
BLOCK_SIZE="${BLOCK_SIZE:-256}"

# DSpark speculative decoding settings (requires a vLLM build with dspark
# support, e.g. the vllm-dspark env or a custom DOCKER_IMAGE).
SPEC_METHOD="${SPEC_METHOD:-dspark}"
SPEC_TOKENS="${SPEC_TOKENS:-5}"
TOKENIZER_MODE="${TOKENIZER_MODE:-deepseek_v4}"
REASONING_PARSER="${REASONING_PARSER:-deepseek_v4}"

# Deployment switch. 0 = native vllm venv, 1 = Docker.
USE_DOCKER="${USE_DOCKER:-1}"
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"

# Engine readiness timeout inside the server (seconds). The DSpark draft model
# compiles ~140 TileLang kernels plus CUDA graphs on first start, which far
# exceeds vLLM's 600s default (VLLM_ENGINE_READY_TIMEOUT_S). 3600s gives
# enough headroom for a cold cache.
VLLM_ENGINE_READY_TIMEOUT_S="${VLLM_ENGINE_READY_TIMEOUT_S:-3600}"

# Health-check retries in the start scripts (5s interval per retry).
# 720 * 5s = 60min, matching VLLM_ENGINE_READY_TIMEOUT_S above.
HEALTH_CHECK_RETRIES="${HEALTH_CHECK_RETRIES:-720}"

# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving,
# so the client runs inside the SGLang image and targets the vLLM backend via the
# OpenAI-compatible API.
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"

# Dataset used by sglang.bench_serving --dataset-name random.
# The random sampler needs a ShareGPT-style JSON file locally; it falls back to
# downloading from HuggingFace, which usually fails on offline H20 nodes.
DATASET_PATH="${DATASET_PATH:-/data1/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}"

# Matrix and concurrency rules are defined in matrix.json by default.
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
MATRIX_MODE="${MATRIX_MODE:-Y}"

# Sampling density for concurrency.
# 0 = use the default heuristic in generate_scenarios.py (6-8 points).
# 2 = only test the low and high endpoints.
export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}"

# Per-scenario timeout to avoid hangs (seconds).
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"

# GPU memory sampling interval (seconds).
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"

# Dry-run mode: if 1, only log the server args and scenario plan without starting
# any server or sending requests.
DRY_RUN="${DRY_RUN:-0}"

# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios.
GRID_LIMIT="${GRID_LIMIT:-0}"
