e72e34c2f7
- Remove USE_FAST_BFP16_MOE=1 which caused torch parameter assignment crash in fresh containers for w8a8_int8 mode. - Mount model path at its host absolute path inside the container so the bench client tokenizer can load it regardless of server --model-path. - Default WARMUP to 0 and NUM_PROMPTS to 10 for a fast smoke test. - Default to a single scenario; document how to run a full grid. - Update experiment README with PLATFORM env var and override examples.
37 lines
1.4 KiB
Bash
37 lines
1.4 KiB
Bash
# Experiment-level configuration for dsv4_p800_sglang
|
|
# All values can be overridden via environment variables.
|
|
|
|
EXPERIMENT="${EXPERIMENT:-dsv4_p800_sglang}"
|
|
MODEL_NAME="${MODEL_NAME:-DeepSeek-V4-Flash-INT8}"
|
|
MODEL_PATH="${MODEL_PATH:-/data1/models/DeepSeek-V4-Flash-INT8}"
|
|
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-deepseek-v4-flash-int8}"
|
|
PORT="${PORT:-30000}"
|
|
BACKEND="${BACKEND:-sglang}"
|
|
ENGINE="${ENGINE:-sglang-xpu}"
|
|
DATASET="${DATASET:-random}"
|
|
# Path to a local ShareGPT-style dataset inside the container.
|
|
# Required when HF Hub is offline. The repo ships a dummy dataset under
|
|
# platforms/patches/kunlun_p800/dummy_sharegpt.json; for managed mode it is
|
|
# mounted to /workspace/dummy_sharegpt.json.
|
|
DATASET_PATH="${DATASET_PATH:-/workspace/dummy_sharegpt.json}"
|
|
WARMUP="${WARMUP:-0}"
|
|
# Default to a small number of prompts for a quick smoke test. Increase to 512
|
|
# or more for a real throughput measurement.
|
|
NUM_PROMPTS="${NUM_PROMPTS:-10}"
|
|
|
|
# Server launch mode for P800 SGLang.
|
|
# - "fp8" : non-INT8 DeepSeek-V4-Flash checkpoint
|
|
# - "w8a8_int8": DeepSeek-V4-Flash-INT8 checkpoint
|
|
SERVER_MODE="${SERVER_MODE:-w8a8_int8}"
|
|
|
|
# Scenario list: "concurrency input_len output_len" (one per line).
|
|
# Default to a single smoke-test scenario for fast validation. Override to run
|
|
# a full grid, e.g.:
|
|
# SCENARIOS=("32 512 256" "128 512 256" "256 512 256" "512 512 256")
|
|
if [[ -z "${SCENARIOS:-}" ]]; then
|
|
SCENARIOS=(
|
|
"32 512 256"
|
|
)
|
|
fi
|
|
|