fix P800 bench_serving P95
This commit is contained in:
yy-fighting
parent
fd03460e19
commit
58ed6ea633
@ -17,8 +17,10 @@ Differences from the H20 version:
|
||||
- Server runs in the P800 vendor image with the proven INT8 launch args
|
||||
(NSA attention backend, `w8a8_int8` quantization);
|
||||
the benchmark client runs inside the same container via `docker exec`.
|
||||
- ISL is capped at 131072 and concurrency ranges are lower, because P800
|
||||
INT8 sustains at most ~131k input tokens and far less concurrency.
|
||||
- ISL is capped at 131072 because P800 INT8 sustains at most ~131k input
|
||||
tokens; concurrency ranges match the H20 matrix (up to 128 at ISL=1024).
|
||||
Scenarios beyond server capacity are recorded as failed/skipped and the
|
||||
run moves on.
|
||||
- XPU memory sampling uses `xpu-smi` instead of `nvidia-smi`.
|
||||
|
||||
`--dp-size` support on this image was probed empirically (2026-07-16,
|
||||
|
||||
@ -6,9 +6,10 @@
|
||||
# PLATEAU_PATIENCE consecutive points.
|
||||
|
||||
SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
|
||||
# The server-side max-running-requests cap has been removed; 32 is kept as
|
||||
# the search upper bound to stay comparable with earlier runs.
|
||||
SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-32}"
|
||||
# No server-side max-running-requests cap is set; aligned with the H20
|
||||
# adaptive search upper bound. Points beyond server capacity are recorded
|
||||
# as failed and the search moves on.
|
||||
SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}"
|
||||
SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}"
|
||||
NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"
|
||||
|
||||
|
||||
@ -0,0 +1,5 @@
|
||||
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":40.23911515297368,"request_tps":1.988115287721181,"input_tps":2035.8300546264893,"output_tps":254.47875682831116,"total_tps":2290.3088114548004,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1052.9030304169282,"ttft_p95_ms":0.0,"ttft_p99_ms":5166.6259043244645,"tpot_p50_ms":49.23821682675089,"tpot_p95_ms":0.0,"tpot_p99_ms":54.25038263861706,"e2e_p50_ms":7454.8611105419695,"e2e_p95_ms":0.0,"e2e_p99_ms":11466.465564270038,"itl_p50_ms":37.7943164203316,"itl_p95_ms":63.24442769400776,"itl_p99_ms":348.378855206538,"validation_errors":[],"timestamp":"2026-07-16T07:20:26+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/tp4_dp2/raw_outputs/sglang_adaptive_c16_i1024_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/tp4_dp2/logs/sglang_c16_i1024_o128_a1.log"}
|
||||
{"status":"COMPLETED","completed":160,"failed":0,"duration_s":50.54340630490333,"request_tps":3.165595904534001,"input_tps":3241.570206242817,"output_tps":405.19627578035215,"total_tps":3646.7664820231694,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":2105.71797657758,"ttft_p95_ms":0.0,"ttft_p99_ms":3751.428431791719,"tpot_p50_ms":58.03926556653482,"tpot_p95_ms":0.0,"tpot_p99_ms":78.91586898356734,"e2e_p50_ms":10047.10013512522,"e2e_p95_ms":0.0,"e2e_p99_ms":11669.661067568231,"itl_p50_ms":44.176343944855034,"itl_p95_ms":68.84024657774717,"itl_p99_ms":453.6125026480295,"validation_errors":[],"timestamp":"2026-07-16T07:21:38+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":59.225973,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/tp4_dp2/raw_outputs/sglang_adaptive_c32_i1024_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/tp4_dp2/logs/sglang_c32_i1024_o128_a1.log"}
|
||||
{"status":"COMPLETED","completed":240,"failed":0,"duration_s":63.502394546056166,"request_tps":3.7793850407630853,"input_tps":3870.0902817413994,"output_tps":483.7612852176749,"total_tps":4353.851566959074,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":3862.513850675896,"ttft_p95_ms":0.0,"ttft_p99_ms":5555.822190991602,"tpot_p50_ms":67.70696836674043,"tpot_p95_ms":0.0,"tpot_p99_ms":93.07615396471353,"e2e_p50_ms":12500.39366458077,"e2e_p95_ms":0.0,"e2e_p99_ms":16880.625398831908,"itl_p50_ms":49.83529110904783,"itl_p95_ms":74.06742639141157,"itl_p99_ms":259.87648053327575,"validation_errors":[],"timestamp":"2026-07-16T07:23:03+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":19.389371,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/tp4_dp2/raw_outputs/sglang_adaptive_c48_i1024_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/tp4_dp2/logs/sglang_c48_i1024_o128_a1.log"}
|
||||
{"status":"COMPLETED","completed":320,"failed":0,"duration_s":85.09563249303028,"request_tps":3.76047501646115,"input_tps":3850.7264168562174,"output_tps":481.3408021070272,"total_tps":4332.067218963245,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":5960.168475518003,"ttft_p95_ms":0.0,"ttft_p99_ms":18375.722452029586,"tpot_p50_ms":65.91272115795397,"tpot_p95_ms":0.0,"tpot_p99_ms":185.4429571080667,"e2e_p50_ms":14741.17284908425,"e2e_p95_ms":0.0,"e2e_p99_ms":29969.413060459774,"itl_p50_ms":53.08219196740538,"itl_p95_ms":75.85310137365013,"itl_p99_ms":249.40574671840295,"validation_errors":[],"timestamp":"2026-07-16T07:24:49+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":64,"num_prompts":320,"warmup_requests":64,"attempt":1,"gain_pct":-0.500347,"plateau_streak":1,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/tp4_dp2/raw_outputs/sglang_adaptive_c64_i1024_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/tp4_dp2/logs/sglang_c64_i1024_o128_a1.log"}
|
||||
{"status":"COMPLETED","completed":400,"failed":0,"duration_s":107.8278465711046,"request_tps":3.7096168820938957,"input_tps":3798.647687264149,"output_tps":474.83096090801865,"total_tps":4273.478648172168,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":7867.146757547744,"ttft_p95_ms":0.0,"ttft_p99_ms":19524.02482017642,"tpot_p50_ms":72.21180715004174,"tpot_p95_ms":0.0,"tpot_p99_ms":181.08834505535836,"e2e_p50_ms":16652.380090556107,"e2e_p95_ms":0.0,"e2e_p99_ms":31464.33779285988,"itl_p50_ms":53.27356734778732,"itl_p95_ms":77.84591265954077,"itl_p99_ms":331.16745246108627,"validation_errors":[],"timestamp":"2026-07-16T07:26:59+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":80,"num_prompts":400,"warmup_requests":80,"attempt":1,"gain_pct":-1.352439,"plateau_streak":2,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/tp4_dp2/raw_outputs/sglang_adaptive_c80_i1024_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-064715/tp4_dp2/logs/sglang_c80_i1024_o128_a1.log"}
|
||||
@ -0,0 +1 @@
|
||||
{"timestamp":"2026-07-16T07:26:59+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"status":"COMPLETED","stop_reason":"TPS_PLATEAU","tested_points":5,"search_cap":512,"max_successful_concurrency":80,"saturation_concurrency":64,"stop_probe_concurrency":80,"best_tps_concurrency":48,"best_total_tps":4353.851566959074,"last_total_tps":4273.478648172168}
|
||||
@ -10,7 +10,7 @@
|
||||
"random_range_ratio": 1.0,
|
||||
"search": {
|
||||
"start_concurrency": 16,
|
||||
"max_concurrency": 32,
|
||||
"max_concurrency": 512,
|
||||
"multiplier": 2,
|
||||
"num_prompts_multiplier": 5,
|
||||
"min_tps_gain_pct": 2.0,
|
||||
|
||||
@ -0,0 +1,25 @@
|
||||
{
|
||||
"status": "COMPLETED",
|
||||
"completed": 80,
|
||||
"failed": 0,
|
||||
"duration_s": 40.23911515297368,
|
||||
"request_tps": 1.988115287721181,
|
||||
"input_tps": 2035.8300546264893,
|
||||
"output_tps": 254.47875682831116,
|
||||
"total_tps": 2290.3088114548004,
|
||||
"mean_input_tokens": 1024.0,
|
||||
"mean_output_tokens": 128.0,
|
||||
"ttft_p50_ms": 1052.9030304169282,
|
||||
"ttft_p95_ms": 0.0,
|
||||
"ttft_p99_ms": 5166.6259043244645,
|
||||
"tpot_p50_ms": 49.23821682675089,
|
||||
"tpot_p95_ms": 0.0,
|
||||
"tpot_p99_ms": 54.25038263861706,
|
||||
"e2e_p50_ms": 7454.8611105419695,
|
||||
"e2e_p95_ms": 0.0,
|
||||
"e2e_p99_ms": 11466.465564270038,
|
||||
"itl_p50_ms": 37.7943164203316,
|
||||
"itl_p95_ms": 63.24442769400776,
|
||||
"itl_p99_ms": 348.378855206538,
|
||||
"validation_errors": []
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
{
|
||||
"status": "COMPLETED",
|
||||
"completed": 80,
|
||||
"failed": 0,
|
||||
"duration_s": 57.74114781501703,
|
||||
"request_tps": 1.3854937601222053,
|
||||
"input_tps": 1418.7456103651382,
|
||||
"output_tps": 354.68640259128455,
|
||||
"total_tps": 1773.4320129564228,
|
||||
"mean_input_tokens": 1024.0,
|
||||
"mean_output_tokens": 256.0,
|
||||
"ttft_p50_ms": 603.6663899431005,
|
||||
"ttft_p95_ms": 0.0,
|
||||
"ttft_p99_ms": 1904.6510633290745,
|
||||
"tpot_p50_ms": 42.63468751565133,
|
||||
"tpot_p95_ms": 0.0,
|
||||
"tpot_p99_ms": 45.214616991625185,
|
||||
"e2e_p50_ms": 11689.894185517915,
|
||||
"e2e_p95_ms": 0.0,
|
||||
"e2e_p99_ms": 12117.081098477356,
|
||||
"itl_p50_ms": 35.19838349893689,
|
||||
"itl_p95_ms": 57.88392436224967,
|
||||
"itl_p99_ms": 250.10306392796338,
|
||||
"validation_errors": []
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
{
|
||||
"status": "COMPLETED",
|
||||
"completed": 160,
|
||||
"failed": 0,
|
||||
"duration_s": 50.54340630490333,
|
||||
"request_tps": 3.165595904534001,
|
||||
"input_tps": 3241.570206242817,
|
||||
"output_tps": 405.19627578035215,
|
||||
"total_tps": 3646.7664820231694,
|
||||
"mean_input_tokens": 1024.0,
|
||||
"mean_output_tokens": 128.0,
|
||||
"ttft_p50_ms": 2105.71797657758,
|
||||
"ttft_p95_ms": 0.0,
|
||||
"ttft_p99_ms": 3751.428431791719,
|
||||
"tpot_p50_ms": 58.03926556653482,
|
||||
"tpot_p95_ms": 0.0,
|
||||
"tpot_p99_ms": 78.91586898356734,
|
||||
"e2e_p50_ms": 10047.10013512522,
|
||||
"e2e_p95_ms": 0.0,
|
||||
"e2e_p99_ms": 11669.661067568231,
|
||||
"itl_p50_ms": 44.176343944855034,
|
||||
"itl_p95_ms": 68.84024657774717,
|
||||
"itl_p99_ms": 453.6125026480295,
|
||||
"validation_errors": []
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
{
|
||||
"status": "COMPLETED",
|
||||
"completed": 160,
|
||||
"failed": 0,
|
||||
"duration_s": 70.48552284296602,
|
||||
"request_tps": 2.2699696837953853,
|
||||
"input_tps": 2324.4489562064746,
|
||||
"output_tps": 581.1122390516186,
|
||||
"total_tps": 2905.5611952580934,
|
||||
"mean_input_tokens": 1024.0,
|
||||
"mean_output_tokens": 256.0,
|
||||
"ttft_p50_ms": 1804.3314524693415,
|
||||
"ttft_p95_ms": 0.0,
|
||||
"ttft_p99_ms": 3630.1654690341093,
|
||||
"tpot_p50_ms": 47.34475767221667,
|
||||
"tpot_p95_ms": 0.0,
|
||||
"tpot_p99_ms": 55.438701826604266,
|
||||
"e2e_p50_ms": 13931.560064083897,
|
||||
"e2e_p95_ms": 0.0,
|
||||
"e2e_p99_ms": 15806.807921521831,
|
||||
"itl_p50_ms": 38.56221307069063,
|
||||
"itl_p95_ms": 58.92605511471629,
|
||||
"itl_p99_ms": 280.3189510037191,
|
||||
"validation_errors": []
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
{
|
||||
"status": "COMPLETED",
|
||||
"completed": 240,
|
||||
"failed": 0,
|
||||
"duration_s": 63.502394546056166,
|
||||
"request_tps": 3.7793850407630853,
|
||||
"input_tps": 3870.0902817413994,
|
||||
"output_tps": 483.7612852176749,
|
||||
"total_tps": 4353.851566959074,
|
||||
"mean_input_tokens": 1024.0,
|
||||
"mean_output_tokens": 128.0,
|
||||
"ttft_p50_ms": 3862.513850675896,
|
||||
"ttft_p95_ms": 0.0,
|
||||
"ttft_p99_ms": 5555.822190991602,
|
||||
"tpot_p50_ms": 67.70696836674043,
|
||||
"tpot_p95_ms": 0.0,
|
||||
"tpot_p99_ms": 93.07615396471353,
|
||||
"e2e_p50_ms": 12500.39366458077,
|
||||
"e2e_p95_ms": 0.0,
|
||||
"e2e_p99_ms": 16880.625398831908,
|
||||
"itl_p50_ms": 49.83529110904783,
|
||||
"itl_p95_ms": 74.06742639141157,
|
||||
"itl_p99_ms": 259.87648053327575,
|
||||
"validation_errors": []
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
{
|
||||
"status": "COMPLETED",
|
||||
"completed": 320,
|
||||
"failed": 0,
|
||||
"duration_s": 85.09563249303028,
|
||||
"request_tps": 3.76047501646115,
|
||||
"input_tps": 3850.7264168562174,
|
||||
"output_tps": 481.3408021070272,
|
||||
"total_tps": 4332.067218963245,
|
||||
"mean_input_tokens": 1024.0,
|
||||
"mean_output_tokens": 128.0,
|
||||
"ttft_p50_ms": 5960.168475518003,
|
||||
"ttft_p95_ms": 0.0,
|
||||
"ttft_p99_ms": 18375.722452029586,
|
||||
"tpot_p50_ms": 65.91272115795397,
|
||||
"tpot_p95_ms": 0.0,
|
||||
"tpot_p99_ms": 185.4429571080667,
|
||||
"e2e_p50_ms": 14741.17284908425,
|
||||
"e2e_p95_ms": 0.0,
|
||||
"e2e_p99_ms": 29969.413060459774,
|
||||
"itl_p50_ms": 53.08219196740538,
|
||||
"itl_p95_ms": 75.85310137365013,
|
||||
"itl_p99_ms": 249.40574671840295,
|
||||
"validation_errors": []
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
{
|
||||
"status": "COMPLETED",
|
||||
"completed": 400,
|
||||
"failed": 0,
|
||||
"duration_s": 107.8278465711046,
|
||||
"request_tps": 3.7096168820938957,
|
||||
"input_tps": 3798.647687264149,
|
||||
"output_tps": 474.83096090801865,
|
||||
"total_tps": 4273.478648172168,
|
||||
"mean_input_tokens": 1024.0,
|
||||
"mean_output_tokens": 128.0,
|
||||
"ttft_p50_ms": 7867.146757547744,
|
||||
"ttft_p95_ms": 0.0,
|
||||
"ttft_p99_ms": 19524.02482017642,
|
||||
"tpot_p50_ms": 72.21180715004174,
|
||||
"tpot_p95_ms": 0.0,
|
||||
"tpot_p99_ms": 181.08834505535836,
|
||||
"e2e_p50_ms": 16652.380090556107,
|
||||
"e2e_p95_ms": 0.0,
|
||||
"e2e_p99_ms": 31464.33779285988,
|
||||
"itl_p50_ms": 53.27356734778732,
|
||||
"itl_p95_ms": 77.84591265954077,
|
||||
"itl_p99_ms": 331.16745246108627,
|
||||
"validation_errors": []
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":41.2688269931823,"request_tps":1.9385091806272121,"input_tps":1985.0334009622652,"output_tps":248.12917512028315,"total_tps":2233.1625760825486,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1839.1637965105474,"ttft_p95_ms":5156.00185089279,"ttft_p99_ms":5158.878551644739,"tpot_p50_ms":47.26517358684809,"tpot_p95_ms":54.75502148450271,"tpot_p99_ms":55.868444815663054,"e2e_p50_ms":7647.177290520631,"e2e_p95_ms":11683.607439068146,"e2e_p99_ms":11687.237907222006,"itl_p50_ms":38.27845409978181,"itl_p95_ms":61.400388937909156,"itl_p99_ms":83.85505519807339,"validation_errors":[],"timestamp":"2026-07-16T07:48:44+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-074243/tp4_dp2/raw_outputs/sglang_adaptive_c16_i1024_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/adaptive_results/adaptive_20260716-074243/tp4_dp2/logs/sglang_c16_i1024_o128_a1.log"}
|
||||
@ -0,0 +1 @@
|
||||
{"timestamp":"2026-07-16T07:48:44+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":2233.1625760825486,"last_total_tps":2233.1625760825486}
|
||||
@ -0,0 +1,22 @@
|
||||
{
|
||||
"experiment": "dsv4_p800_sglang_tp_dp_matrix",
|
||||
"engine": "sglang",
|
||||
"run_id": "adaptive_20260716-074243",
|
||||
"model": "/data1/models/DeepSeek-V4-Flash-INT8",
|
||||
"hardware": "8x Kunlun P800 XPU",
|
||||
"matrix": "/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/matrix.json",
|
||||
"dataset": "random",
|
||||
"tokenize_prompt": false,
|
||||
"random_range_ratio": 1.0,
|
||||
"search": {
|
||||
"start_concurrency": 16,
|
||||
"max_concurrency": 512,
|
||||
"multiplier": 2,
|
||||
"num_prompts_multiplier": 5,
|
||||
"min_tps_gain_pct": 2.0,
|
||||
"plateau_patience": 2,
|
||||
"warmup_max_requests": 0,
|
||||
"ttft_slo_ms": 4000.0,
|
||||
"enable_ttft_slo_stop": 1
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
mark input_len output_len
|
||||
Y 1024 128
|
||||
Y 1024 256
|
||||
Y 1024 512
|
||||
Y 1024 1024
|
||||
Y 1024 2048
|
||||
Y 1024 4096
|
||||
Y 4096 128
|
||||
Y 4096 256
|
||||
Y 4096 512
|
||||
Y 4096 1024
|
||||
Y 4096 2048
|
||||
Y 4096 4096
|
||||
Y 16384 128
|
||||
Y 16384 256
|
||||
Y 16384 512
|
||||
Y 16384 1024
|
||||
Y 16384 2048
|
||||
Y 65536 128
|
||||
Y 65536 256
|
||||
Y 65536 512
|
||||
Y 65536 1024
|
||||
Y 131072 128
|
||||
Y 131072 256
|
||||
Y 131072 512
|
||||
|
@ -0,0 +1 @@
|
||||
--host 0.0.0.0 --port 30014 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static 0.8 --tensor-parallel-size 2 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 140000 --dp-size 4
|
||||
@ -0,0 +1,25 @@
|
||||
{
|
||||
"status": "COMPLETED",
|
||||
"completed": 80,
|
||||
"failed": 0,
|
||||
"duration_s": 41.2688269931823,
|
||||
"request_tps": 1.9385091806272121,
|
||||
"input_tps": 1985.0334009622652,
|
||||
"output_tps": 248.12917512028315,
|
||||
"total_tps": 2233.1625760825486,
|
||||
"mean_input_tokens": 1024.0,
|
||||
"mean_output_tokens": 128.0,
|
||||
"ttft_p50_ms": 1839.1637965105474,
|
||||
"ttft_p95_ms": 5156.00185089279,
|
||||
"ttft_p99_ms": 5158.878551644739,
|
||||
"tpot_p50_ms": 47.26517358684809,
|
||||
"tpot_p95_ms": 54.75502148450271,
|
||||
"tpot_p99_ms": 55.868444815663054,
|
||||
"e2e_p50_ms": 7647.177290520631,
|
||||
"e2e_p95_ms": 11683.607439068146,
|
||||
"e2e_p99_ms": 11687.237907222006,
|
||||
"itl_p50_ms": 38.27845409978181,
|
||||
"itl_p95_ms": 61.400388937909156,
|
||||
"itl_p99_ms": 83.85505519807339,
|
||||
"validation_errors": []
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
--host 0.0.0.0 --port 30014 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static 0.8 --tensor-parallel-size 4 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 140000 --dp-size 2
|
||||
@ -55,12 +55,13 @@ MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.8}"
|
||||
|
||||
DOCKER_IMAGE="${DOCKER_IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202}"
|
||||
|
||||
# Random dataset mounted into the container at /workspace/dummy_sharegpt.json.
|
||||
# DATASET_PATH is the host copy (used by the adaptive search for sanity
|
||||
# checks); the benchmark client always references CONTAINER_DATASET_PATH.
|
||||
PATCH_ROOT="${PATCH_ROOT:-/data1/yy/sskj/platforms/patches/kunlun_p800}"
|
||||
DATASET_PATH="${DATASET_PATH:-${PATCH_ROOT}/dummy_sharegpt.json}"
|
||||
CONTAINER_DATASET_PATH="${CONTAINER_DATASET_PATH:-/workspace/dummy_sharegpt.json}"
|
||||
# ShareGPT seed dataset for the random workload. DATASET_PATH is the host
|
||||
# copy (also used by the adaptive search for capacity checks); it is mounted
|
||||
# into the container at CONTAINER_DATASET_PATH, which is what the benchmark
|
||||
# client references. Default matches the H20 experiments; the file is
|
||||
# gitignored, download it into /data1/yy/sskj/datasets/ first.
|
||||
DATASET_PATH="${DATASET_PATH:-/data1/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}"
|
||||
CONTAINER_DATASET_PATH="${CONTAINER_DATASET_PATH:-/workspace/ShareGPT_V3_unfiltered_cleaned_split.json}"
|
||||
|
||||
# Matrix and concurrency rules are defined in matrix.json by default.
|
||||
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
{
|
||||
"comment": "ISL/OSL matrix for dsv4_p800_sglang_tp_dp_matrix. Y=must test, P=optional (record skipped on failure), N=skip.",
|
||||
"mode": "Y",
|
||||
"notes": "P800 INT8 sustains at most ~131k input tokens, so ISL stops at 131072 and concurrency ranges are much lower than the H20 matrix.",
|
||||
"notes": "P800 INT8 sustains at most ~131k input tokens, so ISL stops at 131072. Concurrency ranges match the H20 matrix; scenarios that exceed server capacity are recorded as failed/skipped and the run moves on.",
|
||||
"matrix": {
|
||||
"1024": {
|
||||
"128": "Y",
|
||||
@ -45,10 +45,10 @@
|
||||
}
|
||||
},
|
||||
"concurrency": {
|
||||
"1024": { "low": 1, "high": 32 },
|
||||
"4096": { "low": 1, "high": 16 },
|
||||
"16384": { "low": 1, "high": 8 },
|
||||
"65536": { "low": 1, "high": 4 },
|
||||
"131072": { "low": 1, "high": 2 }
|
||||
"1024": { "low": 1, "high": 128 },
|
||||
"4096": { "low": 1, "high": 64 },
|
||||
"16384": { "low": 1, "high": 32 },
|
||||
"65536": { "low": 1, "high": 8 },
|
||||
"131072": { "low": 1, "high": 4 }
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -112,8 +112,23 @@ server_cmd_b64=$(printf '%s' "$server_cmd" | base64 -w0)
|
||||
|
||||
patch_mounts=(
|
||||
-v "${PATCH_ROOT}/nic_priority_matrix_test.json:/workspace/nic_priority_matrix_test.json:ro"
|
||||
-v "${PATCH_ROOT}/dummy_sharegpt.json:/workspace/dummy_sharegpt.json:ro"
|
||||
)
|
||||
# Patch the image's sglang.bench_serving to also report TTFT/TPOT/E2E P95.
|
||||
# The P800 image only ships median/P99 for those metrics, and the adaptive
|
||||
# search reads p95_ttft_ms for its TTFT SLO stop. The container is recreated
|
||||
# per TP/DP config, so the patch must be mounted at every `docker run`.
|
||||
BENCH_SERVING_PATCH="${SCRIPT_DIR}/patches/bench_serving.py"
|
||||
if [[ -f "${BENCH_SERVING_PATCH}" ]]; then
|
||||
patch_mounts+=(
|
||||
-v "${BENCH_SERVING_PATCH}:/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/sglang/bench_serving.py:ro"
|
||||
)
|
||||
fi
|
||||
# Mount the ShareGPT seed dataset only when it exists on the host, so that
|
||||
# BENCH_DATASET_NAME=random-ids (no seed needed) never makes docker create a
|
||||
# stray directory at DATASET_PATH.
|
||||
if [[ -f "${DATASET_PATH}" ]]; then
|
||||
patch_mounts+=( -v "${DATASET_PATH}:${CONTAINER_DATASET_PATH}:ro" )
|
||||
fi
|
||||
|
||||
echo "=== Starting P800 SGLang INT8 server in Docker (TP=${TP}, DP=${DP}) ==="
|
||||
echo "Image: ${DOCKER_IMAGE}"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user