fix(p800): managed-mode server startup and smoke-test defaults
- Remove USE_FAST_BFP16_MOE=1 which caused torch parameter assignment crash in fresh containers for w8a8_int8 mode. - Mount model path at its host absolute path inside the container so the bench client tokenizer can load it regardless of server --model-path. - Default WARMUP to 0 and NUM_PROMPTS to 10 for a fast smoke test. - Default to a single scenario; document how to run a full grid. - Update experiment README with PLATFORM env var and override examples.
This commit is contained in:
parent
227ef603a6
commit
e72e34c2f7
@ -5,15 +5,27 @@ Kunlun P800 XPU + SGLang + `DeepSeek-V4-Flash-INT8` benchmark experiment.
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Run all configured scenarios (starts server, runs benchmark, stops server)
|
||||
bash experiments/dsv4_p800_sglang/run_bench.sh
|
||||
# Run the default smoke-test scenario (starts server, runs benchmark, stops server)
|
||||
PLATFORM=kunlun_p800 bash experiments/dsv4_p800_sglang/run_bench.sh
|
||||
|
||||
# Reuse an already-running server
|
||||
SKIP_MANAGE_SERVER=1 bash experiments/dsv4_p800_sglang/run_bench.sh
|
||||
SKIP_MANAGE_SERVER=1 PLATFORM=kunlun_p800 bash experiments/dsv4_p800_sglang/run_bench.sh
|
||||
```
|
||||
|
||||
Results land in `experiments/dsv4_p800_sglang/results/<RUN_ID>/`.
|
||||
|
||||
## Configuration
|
||||
|
||||
Edit `config.env` or override via environment variables:
|
||||
|
||||
```bash
|
||||
# Run a full throughput grid instead of the default single smoke-test scenario
|
||||
SCENARIOS=("32 512 256" "128 512 256" "256 512 256" "512 512 256") \
|
||||
NUM_PROMPTS=512 \
|
||||
PLATFORM=kunlun_p800 \
|
||||
bash experiments/dsv4_p800_sglang/run_bench.sh
|
||||
```
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose |
|
||||
|
||||
@ -14,22 +14,23 @@ DATASET="${DATASET:-random}"
|
||||
# platforms/patches/kunlun_p800/dummy_sharegpt.json; for managed mode it is
|
||||
# mounted to /workspace/dummy_sharegpt.json.
|
||||
DATASET_PATH="${DATASET_PATH:-/workspace/dummy_sharegpt.json}"
|
||||
WARMUP="${WARMUP:-100}"
|
||||
NUM_PROMPTS="${NUM_PROMPTS:-512}"
|
||||
WARMUP="${WARMUP:-0}"
|
||||
# Default to a small number of prompts for a quick smoke test. Increase to 512
|
||||
# or more for a real throughput measurement.
|
||||
NUM_PROMPTS="${NUM_PROMPTS:-10}"
|
||||
|
||||
# Server launch mode for P800 SGLang.
|
||||
# - "fp8" : non-INT8 DeepSeek-V4-Flash checkpoint
|
||||
# - "w8a8_int8": DeepSeek-V4-Flash-INT8 checkpoint
|
||||
SERVER_MODE="${SERVER_MODE:-w8a8_int8}"
|
||||
|
||||
# Scenario list: "concurrency input_len output_len" (one per line)
|
||||
# Override example: SCENARIOS=("32 512 256")
|
||||
# Scenario list: "concurrency input_len output_len" (one per line).
|
||||
# Default to a single smoke-test scenario for fast validation. Override to run
|
||||
# a full grid, e.g.:
|
||||
# SCENARIOS=("32 512 256" "128 512 256" "256 512 256" "512 512 256")
|
||||
if [[ -z "${SCENARIOS:-}" ]]; then
|
||||
SCENARIOS=(
|
||||
"32 512 256"
|
||||
"128 512 256"
|
||||
"256 512 256"
|
||||
"512 512 256"
|
||||
)
|
||||
fi
|
||||
|
||||
|
||||
@ -101,7 +101,6 @@ docker_server_start() {
|
||||
-e XINFER_QUANT_SDNN=1
|
||||
-e XSGL_USE_MOE_SIGMOID_GROUP_TOPK_NORM=1
|
||||
-e XSGL_EARLY_FIRST_TOKEN=1
|
||||
-e USE_FAST_BFP16_MOE=1
|
||||
-e XSGL_ENABLE_TGEMM_FP16=1
|
||||
-e SGLANG_ENABLE_SPEC_V2=True
|
||||
-e SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
@ -189,6 +188,7 @@ EOF
|
||||
--ipc host \
|
||||
${device_args} \
|
||||
-v "${model_path}:/models:ro" \
|
||||
-v "${model_path}:${model_path}:ro" \
|
||||
"${patch_mounts[@]}" \
|
||||
"${env_args[@]}" \
|
||||
"${image}" \
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user