3694614dd9
Add experiments/dsv4_p800_max_context_length/ to find the longest input context that P800 SGLang INT8 can serve for DeepSeek-V4-Flash-INT8. - Docker-based server start with dynamic --context-length. - Single-request, single-output-token probes for each candidate length. - Records success/failure, server args, and metrics per length. - Generates results.json and report.md summarizing the max supported length. - Smoke-tested at 4096 tokens successfully.
76 lines
2.7 KiB
JSON
76 lines
2.7 KiB
JSON
{
|
|
"metadata": {
|
|
"experiment": "dsv4_p800_max_context_length",
|
|
"run_id": "20260709-032422",
|
|
"timestamp": "2026-07-09T03:24:22+00:00",
|
|
"model": "/data1/models/DeepSeek-V4-Flash-INT8",
|
|
"backend": "sglang",
|
|
"engine": "sglang-xpu",
|
|
"hardware": "8x Kunlun P800 XPU",
|
|
"accelerator": "Kunlun P800 XPU",
|
|
"chip": "kunlun_p800",
|
|
"script": "experiments/dsv4_p800_max_context_length/run_bench.sh",
|
|
"env": "",
|
|
"git_commit": "1d1c79d",
|
|
"git_dirty": "dirty",
|
|
"description": "P800 max context length exploration for DeepSeek-V4-Flash-INT8"
|
|
},
|
|
"config": {
|
|
"tp": 8,
|
|
"ep": 8,
|
|
"xpu_visible_devices": "0,1,2,3,4,5,6,7",
|
|
"output_len": 1,
|
|
"num_prompts": 1,
|
|
"max_concurrency": 1,
|
|
"request_rate": 10000,
|
|
"context_pad": 1024
|
|
},
|
|
"scenarios": [
|
|
{
|
|
"backend": "sglang",
|
|
"target_len": 4096,
|
|
"max_context_len": 5120,
|
|
"success": true,
|
|
"server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 5120 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 5120",
|
|
"metrics": {
|
|
"success_count": 1,
|
|
"total_count": 1,
|
|
"duration_s": 3.3442696970305406,
|
|
"request_throughput": 0.2990189460162034,
|
|
"input_token_throughput": 1224.781602882369,
|
|
"output_token_throughput": 0.2990189460162034,
|
|
"total_input_tokens": 4096,
|
|
"total_output_tokens": 1,
|
|
"e2e_ms": {
|
|
"mean": 231.34461202425882,
|
|
"p50": 231.34461202425882,
|
|
"p90": 231.34461202425882,
|
|
"p95": 0.0,
|
|
"p99": 231.34461202425882
|
|
},
|
|
"ttft_ms": {
|
|
"mean": 231.33470199536532,
|
|
"p50": 231.33470199536532,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 231.33470199536532
|
|
},
|
|
"tpot_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
},
|
|
"itl_ms": {
|
|
"mean": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
"error": null
|
|
}
|
|
]
|
|
} |