feat(bench/pd): 支持 --flush-cache 透传 + PD profile 加 --disable-radix-cache
- sskj.bench: 新增 --flush-cache 参数,透传给 sglang.bench_serving (warmup 后、main run 前 flush 服务器 KV/prefix cache,保证 TTFT/TPOT 测量纯净) - P/D PD profiles: LAUNCH_ARGS 加 --disable-radix-cache(关闭 RadixAttention 前缀缓存)
This commit is contained in:
parent
da0e1b4372
commit
ddf807d458
@ -51,4 +51,4 @@ PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /tmp/flash_kda-0.
|
|||||||
# BOOTSTRAP: 打补丁 → 装 flashkda + mooncake wheel → 按 rank 设 SGLANG_HOST_IP → 启动 decode。
|
# BOOTSTRAP: 打补丁 → 装 flashkda + mooncake wheel → 按 rank 设 SGLANG_HOST_IP → 启动 decode。
|
||||||
BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && pip install /flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl --no-deps -q && pip install /mc_wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-cp312-cp312-manylinux_2_28_x86_64.whl --no-deps -q && export SGLANG_HOST_IP=\"174.1.60.$((5 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}"
|
BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && pip install /flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl --no-deps -q && pip install /mc_wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-cp312-cp312-manylinux_2_28_x86_64.whl --no-deps -q && export SGLANG_HOST_IP=\"174.1.60.$((5 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}"
|
||||||
|
|
||||||
LAUNCH_ARGS="--model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size ${TP} --ep-size 32 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --moe-runner-backend marlin --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --disaggregation-transfer-backend mooncake --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 --disaggregation-mode decode --host 0.0.0.0 --port ${PORT}"
|
LAUNCH_ARGS="--model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size ${TP} --ep-size 32 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --moe-runner-backend marlin --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer --disable-radix-cache --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --disaggregation-transfer-backend mooncake --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 --disaggregation-mode decode --host 0.0.0.0 --port ${PORT}"
|
||||||
|
|||||||
@ -57,4 +57,4 @@ PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /tmp/flash_kda-0.
|
|||||||
# BOOTSTRAP: 打补丁 → 装 flashkda + mooncake wheel → 按 rank 设 SGLANG_HOST_IP → 启动 prefill。
|
# BOOTSTRAP: 打补丁 → 装 flashkda + mooncake wheel → 按 rank 设 SGLANG_HOST_IP → 启动 prefill。
|
||||||
BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && pip install /flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl --no-deps -q && pip install /mc_wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-cp312-cp312-manylinux_2_28_x86_64.whl --no-deps -q && export SGLANG_HOST_IP=\"174.1.60.$((1 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}"
|
BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && pip install /flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl --no-deps -q && pip install /mc_wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-cp312-cp312-manylinux_2_28_x86_64.whl --no-deps -q && export SGLANG_HOST_IP=\"174.1.60.$((1 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}"
|
||||||
|
|
||||||
LAUNCH_ARGS="--model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size ${TP} --ep-size 32 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --moe-runner-backend marlin --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --disaggregation-transfer-backend mooncake --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 --disaggregation-mode prefill --linear-attn-prefill-backend flashkda --host 0.0.0.0 --port ${PORT}"
|
LAUNCH_ARGS="--model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size ${TP} --ep-size 32 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --moe-runner-backend marlin --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer --disable-radix-cache --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --disaggregation-transfer-backend mooncake --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 --disaggregation-mode prefill --linear-attn-prefill-backend flashkda --host 0.0.0.0 --port ${PORT}"
|
||||||
|
|||||||
@ -48,6 +48,7 @@ def _build_parser() -> argparse.ArgumentParser:
|
|||||||
run_parser.add_argument("--output-dir", type=Path, default=None)
|
run_parser.add_argument("--output-dir", type=Path, default=None)
|
||||||
run_parser.add_argument("--timeout", type=int, default=1800)
|
run_parser.add_argument("--timeout", type=int, default=1800)
|
||||||
run_parser.add_argument("--health-wait", type=int, default=600)
|
run_parser.add_argument("--health-wait", type=int, default=600)
|
||||||
|
run_parser.add_argument("--flush-cache", action="store_true", help="Pass --flush-cache to sglang.bench_serving (flush server KV/prefix cache after warmup, before main run)")
|
||||||
run_parser.add_argument("--dry-run", action="store_true")
|
run_parser.add_argument("--dry-run", action="store_true")
|
||||||
return parser
|
return parser
|
||||||
|
|
||||||
@ -198,6 +199,7 @@ def _cmd_run(args: argparse.Namespace, root: Path) -> int:
|
|||||||
container_dataset_path=experiment_env.get("CONTAINER_DATASET_PATH"),
|
container_dataset_path=experiment_env.get("CONTAINER_DATASET_PATH"),
|
||||||
random_range_ratio=float(experiment_env.get("RANDOM_RANGE_RATIO", "1.0")),
|
random_range_ratio=float(experiment_env.get("RANDOM_RANGE_RATIO", "1.0")),
|
||||||
warmup_max_requests=int(experiment_env.get("BENCH_WARMUP_MAX_REQUESTS", "0") or 0),
|
warmup_max_requests=int(experiment_env.get("BENCH_WARMUP_MAX_REQUESTS", "0") or 0),
|
||||||
|
flush_cache=args.flush_cache,
|
||||||
client_mode=client_mode,
|
client_mode=client_mode,
|
||||||
client_image=client_image,
|
client_image=client_image,
|
||||||
client_python=client_python,
|
client_python=client_python,
|
||||||
|
|||||||
@ -24,6 +24,7 @@ class BenchClientOptions:
|
|||||||
container_dataset_path: str | None = None
|
container_dataset_path: str | None = None
|
||||||
random_range_ratio: float = 1.0
|
random_range_ratio: float = 1.0
|
||||||
warmup_max_requests: int = 0
|
warmup_max_requests: int = 0
|
||||||
|
flush_cache: bool = False
|
||||||
client_mode: str = "auto"
|
client_mode: str = "auto"
|
||||||
client_image: str | None = None
|
client_image: str | None = None
|
||||||
client_python: str | None = None
|
client_python: str | None = None
|
||||||
@ -68,6 +69,8 @@ def _bench_args(scenario: dict[str, Any], options: BenchClientOptions) -> list[s
|
|||||||
]
|
]
|
||||||
if options.warmup_max_requests and options.warmup_max_requests > 0:
|
if options.warmup_max_requests and options.warmup_max_requests > 0:
|
||||||
args += ["--warmup-requests", str(options.warmup_max_requests)]
|
args += ["--warmup-requests", str(options.warmup_max_requests)]
|
||||||
|
if options.flush_cache:
|
||||||
|
args += ["--flush-cache"]
|
||||||
|
|
||||||
dataset_path = (
|
dataset_path = (
|
||||||
options.container_dataset_path
|
options.container_dataset_path
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user