From ddf807d4586a03336e5a7f03b660effc7498134b Mon Sep 17 00:00:00 2001 From: shishi Date: Tue, 11 Aug 2026 11:42:43 +0800 Subject: [PATCH] =?UTF-8?q?feat(bench/pd):=20=E6=94=AF=E6=8C=81=20--flush-?= =?UTF-8?q?cache=20=E9=80=8F=E4=BC=A0=20+=20PD=20profile=20=E5=8A=A0=20--d?= =?UTF-8?q?isable-radix-cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - sskj.bench: 新增 --flush-cache 参数，透传给 sglang.bench_serving （warmup 后、main run 前 flush 服务器 KV/prefix cache，保证 TTFT/TPOT 测量纯净） - P/D PD profiles: LAUNCH_ARGS 加 --disable-radix-cache（关闭 RadixAttention 前缀缓存） --- deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env | 2 +- deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env | 2 +- src/sskj/bench/cli.py | 2 ++ src/sskj/bench/runner.py | 3 +++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env b/deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env index 818a720..e3addd7 100644 --- a/deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env +++ b/deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env @@ -51,4 +51,4 @@ PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /tmp/flash_kda-0. # BOOTSTRAP: 打补丁 → 装 flashkda + mooncake wheel → 按 rank 设 SGLANG_HOST_IP → 启动 decode。 BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && pip install /flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl --no-deps -q && pip install /mc_wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-cp312-cp312-manylinux_2_28_x86_64.whl --no-deps -q && export SGLANG_HOST_IP=\"174.1.60.$((5 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}" -LAUNCH_ARGS="--model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size ${TP} --ep-size 32 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --moe-runner-backend marlin --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --disaggregation-transfer-backend mooncake --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 --disaggregation-mode decode --host 0.0.0.0 --port ${PORT}" +LAUNCH_ARGS="--model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size ${TP} --ep-size 32 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --moe-runner-backend marlin --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer --disable-radix-cache --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --disaggregation-transfer-backend mooncake --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 --disaggregation-mode decode --host 0.0.0.0 --port ${PORT}" diff --git a/deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env b/deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env index 43891f5..27d8de1 100644 --- a/deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env +++ b/deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env @@ -57,4 +57,4 @@ PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /tmp/flash_kda-0. # BOOTSTRAP: 打补丁 → 装 flashkda + mooncake wheel → 按 rank 设 SGLANG_HOST_IP → 启动 prefill。 BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && pip install /flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl --no-deps -q && pip install /mc_wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-cp312-cp312-manylinux_2_28_x86_64.whl --no-deps -q && export SGLANG_HOST_IP=\"174.1.60.$((1 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}" -LAUNCH_ARGS="--model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size ${TP} --ep-size 32 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --moe-runner-backend marlin --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --disaggregation-transfer-backend mooncake --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 --disaggregation-mode prefill --linear-attn-prefill-backend flashkda --host 0.0.0.0 --port ${PORT}" +LAUNCH_ARGS="--model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size ${TP} --ep-size 32 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --moe-runner-backend marlin --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer --disable-radix-cache --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --disaggregation-transfer-backend mooncake --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 --disaggregation-mode prefill --linear-attn-prefill-backend flashkda --host 0.0.0.0 --port ${PORT}" diff --git a/src/sskj/bench/cli.py b/src/sskj/bench/cli.py index a93ea27..e395b9f 100644 --- a/src/sskj/bench/cli.py +++ b/src/sskj/bench/cli.py @@ -48,6 +48,7 @@ def _build_parser() -> argparse.ArgumentParser: run_parser.add_argument("--output-dir", type=Path, default=None) run_parser.add_argument("--timeout", type=int, default=1800) run_parser.add_argument("--health-wait", type=int, default=600) + run_parser.add_argument("--flush-cache", action="store_true", help="Pass --flush-cache to sglang.bench_serving (flush server KV/prefix cache after warmup, before main run)") run_parser.add_argument("--dry-run", action="store_true") return parser @@ -198,6 +199,7 @@ def _cmd_run(args: argparse.Namespace, root: Path) -> int: container_dataset_path=experiment_env.get("CONTAINER_DATASET_PATH"), random_range_ratio=float(experiment_env.get("RANDOM_RANGE_RATIO", "1.0")), warmup_max_requests=int(experiment_env.get("BENCH_WARMUP_MAX_REQUESTS", "0") or 0), + flush_cache=args.flush_cache, client_mode=client_mode, client_image=client_image, client_python=client_python, diff --git a/src/sskj/bench/runner.py b/src/sskj/bench/runner.py index 67cc517..9efb686 100644 --- a/src/sskj/bench/runner.py +++ b/src/sskj/bench/runner.py @@ -24,6 +24,7 @@ class BenchClientOptions: container_dataset_path: str | None = None random_range_ratio: float = 1.0 warmup_max_requests: int = 0 + flush_cache: bool = False client_mode: str = "auto" client_image: str | None = None client_python: str | None = None @@ -68,6 +69,8 @@ def _bench_args(scenario: dict[str, Any], options: BenchClientOptions) -> list[s ] if options.warmup_max_requests and options.warmup_max_requests > 0: args += ["--warmup-requests", str(options.warmup_max_requests)] + if options.flush_cache: + args += ["--flush-cache"] dataset_path = ( options.container_dataset_path