fix(p800): managed-mode server startup and smoke-test defaults
- Remove USE_FAST_BFP16_MOE=1 which caused torch parameter assignment crash in fresh containers for w8a8_int8 mode. - Mount model path at its host absolute path inside the container so the bench client tokenizer can load it regardless of server --model-path. - Default WARMUP to 0 and NUM_PROMPTS to 10 for a fast smoke test. - Default to a single scenario; document how to run a full grid. - Update experiment README with PLATFORM env var and override examples.
This commit is contained in:
parent
227ef603a6
commit
e72e34c2f7
@ -5,15 +5,27 @@ Kunlun P800 XPU + SGLang + `DeepSeek-V4-Flash-INT8` benchmark experiment.
|
|||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Run all configured scenarios (starts server, runs benchmark, stops server)
|
# Run the default smoke-test scenario (starts server, runs benchmark, stops server)
|
||||||
bash experiments/dsv4_p800_sglang/run_bench.sh
|
PLATFORM=kunlun_p800 bash experiments/dsv4_p800_sglang/run_bench.sh
|
||||||
|
|
||||||
# Reuse an already-running server
|
# Reuse an already-running server
|
||||||
SKIP_MANAGE_SERVER=1 bash experiments/dsv4_p800_sglang/run_bench.sh
|
SKIP_MANAGE_SERVER=1 PLATFORM=kunlun_p800 bash experiments/dsv4_p800_sglang/run_bench.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
Results land in `experiments/dsv4_p800_sglang/results/<RUN_ID>/`.
|
Results land in `experiments/dsv4_p800_sglang/results/<RUN_ID>/`.
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
Edit `config.env` or override via environment variables:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Run a full throughput grid instead of the default single smoke-test scenario
|
||||||
|
SCENARIOS=("32 512 256" "128 512 256" "256 512 256" "512 512 256") \
|
||||||
|
NUM_PROMPTS=512 \
|
||||||
|
PLATFORM=kunlun_p800 \
|
||||||
|
bash experiments/dsv4_p800_sglang/run_bench.sh
|
||||||
|
```
|
||||||
|
|
||||||
## Files
|
## Files
|
||||||
|
|
||||||
| File | Purpose |
|
| File | Purpose |
|
||||||
|
|||||||
@ -14,22 +14,23 @@ DATASET="${DATASET:-random}"
|
|||||||
# platforms/patches/kunlun_p800/dummy_sharegpt.json; for managed mode it is
|
# platforms/patches/kunlun_p800/dummy_sharegpt.json; for managed mode it is
|
||||||
# mounted to /workspace/dummy_sharegpt.json.
|
# mounted to /workspace/dummy_sharegpt.json.
|
||||||
DATASET_PATH="${DATASET_PATH:-/workspace/dummy_sharegpt.json}"
|
DATASET_PATH="${DATASET_PATH:-/workspace/dummy_sharegpt.json}"
|
||||||
WARMUP="${WARMUP:-100}"
|
WARMUP="${WARMUP:-0}"
|
||||||
NUM_PROMPTS="${NUM_PROMPTS:-512}"
|
# Default to a small number of prompts for a quick smoke test. Increase to 512
|
||||||
|
# or more for a real throughput measurement.
|
||||||
|
NUM_PROMPTS="${NUM_PROMPTS:-10}"
|
||||||
|
|
||||||
# Server launch mode for P800 SGLang.
|
# Server launch mode for P800 SGLang.
|
||||||
# - "fp8" : non-INT8 DeepSeek-V4-Flash checkpoint
|
# - "fp8" : non-INT8 DeepSeek-V4-Flash checkpoint
|
||||||
# - "w8a8_int8": DeepSeek-V4-Flash-INT8 checkpoint
|
# - "w8a8_int8": DeepSeek-V4-Flash-INT8 checkpoint
|
||||||
SERVER_MODE="${SERVER_MODE:-w8a8_int8}"
|
SERVER_MODE="${SERVER_MODE:-w8a8_int8}"
|
||||||
|
|
||||||
# Scenario list: "concurrency input_len output_len" (one per line)
|
# Scenario list: "concurrency input_len output_len" (one per line).
|
||||||
# Override example: SCENARIOS=("32 512 256")
|
# Default to a single smoke-test scenario for fast validation. Override to run
|
||||||
|
# a full grid, e.g.:
|
||||||
|
# SCENARIOS=("32 512 256" "128 512 256" "256 512 256" "512 512 256")
|
||||||
if [[ -z "${SCENARIOS:-}" ]]; then
|
if [[ -z "${SCENARIOS:-}" ]]; then
|
||||||
SCENARIOS=(
|
SCENARIOS=(
|
||||||
"32 512 256"
|
"32 512 256"
|
||||||
"128 512 256"
|
|
||||||
"256 512 256"
|
|
||||||
"512 512 256"
|
|
||||||
)
|
)
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
@ -101,7 +101,6 @@ docker_server_start() {
|
|||||||
-e XINFER_QUANT_SDNN=1
|
-e XINFER_QUANT_SDNN=1
|
||||||
-e XSGL_USE_MOE_SIGMOID_GROUP_TOPK_NORM=1
|
-e XSGL_USE_MOE_SIGMOID_GROUP_TOPK_NORM=1
|
||||||
-e XSGL_EARLY_FIRST_TOKEN=1
|
-e XSGL_EARLY_FIRST_TOKEN=1
|
||||||
-e USE_FAST_BFP16_MOE=1
|
|
||||||
-e XSGL_ENABLE_TGEMM_FP16=1
|
-e XSGL_ENABLE_TGEMM_FP16=1
|
||||||
-e SGLANG_ENABLE_SPEC_V2=True
|
-e SGLANG_ENABLE_SPEC_V2=True
|
||||||
-e SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
-e SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||||
@ -189,6 +188,7 @@ EOF
|
|||||||
--ipc host \
|
--ipc host \
|
||||||
${device_args} \
|
${device_args} \
|
||||||
-v "${model_path}:/models:ro" \
|
-v "${model_path}:/models:ro" \
|
||||||
|
-v "${model_path}:${model_path}:ro" \
|
||||||
"${patch_mounts[@]}" \
|
"${patch_mounts[@]}" \
|
||||||
"${env_args[@]}" \
|
"${env_args[@]}" \
|
||||||
"${image}" \
|
"${image}" \
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user