diff --git a/experiments/dsv4_h200_sglang_vs_vllm/README.md b/experiments/dsv4_h200_sglang_vs_vllm/README.md index 0b5b729..ec71558 100644 --- a/experiments/dsv4_h200_sglang_vs_vllm/README.md +++ b/experiments/dsv4_h200_sglang_vs_vllm/README.md @@ -31,6 +31,15 @@ | 1 | 131072 | 1024 | 2 | | 1 | 200000 | 1024 | 1 | +### ShareGPT 真实对话数据(max_model_len=32k) + +| Concurrency | ShareGPT context len | Output | Num prompts | +|---:|---:|---:|---:| +| 1 | 32768 | 256 | 32 | +| 32 | 32768 | 256 | 128 | + +使用 `datasets/ShareGPT_filtered_chat.json`,`--sharegpt-context-len 32768` 会自动过滤超长请求。 + ## 快速运行 ```bash diff --git a/experiments/dsv4_h200_sglang_vs_vllm/config.env b/experiments/dsv4_h200_sglang_vs_vllm/config.env index ca5e2d8..7d1708f 100644 --- a/experiments/dsv4_h200_sglang_vs_vllm/config.env +++ b/experiments/dsv4_h200_sglang_vs_vllm/config.env @@ -46,6 +46,16 @@ PHASE2_MAX_MODEL_LEN=210000 PHASE2_MAX_NUM_SEQS=2 PHASE2_MAX_RUNNING=2 +# ShareGPT dataset test (uses the same max_model_len as Phase 1). +SHAREGPT_DATASET="${SHAREGPT_DATASET:-/data/user1/yy/datasets/ShareGPT_filtered_chat.json}" +SHAREGPT_CONTEXT_LEN=32768 +if [[ -z "${SHAREGPT_SCENARIOS:-}" ]]; then + declare -a SHAREGPT_SCENARIOS=( + "1 32768 256 32" + "32 32768 256 128" + ) +fi + # Benchmark client always runs from the sglang env so that the same # sglang.bench_serving version is used for both backends. VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" diff --git a/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py b/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py index 9da7392..721394e 100755 --- a/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py +++ b/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py @@ -140,16 +140,27 @@ def main() -> None: if backend is None: raise SystemExit("Could not infer backend from raw outputs") - # Filename: {backend}_phase{1|2}_MMDD_concurrency_inputlen_outputlen.jsonl - pattern = re.compile(rf"^{backend}_phase(\d+)_(\d+)_(\d+)_(\d+)_(\d+)\.jsonl$") - scenarios = [] - for jsonl_path in sorted(raw_dir.glob(f"{backend}_phase*.jsonl")): - m = pattern.match(jsonl_path.name) - if not m: + for jsonl_path in sorted(raw_dir.glob(f"{backend}_*.jsonl")): + parts = jsonl_path.stem.split("_") + if len(parts) < 5: + continue + + label = parts[1] + if label.startswith("phase"): + phase = label + dataset = "random" + elif label == "sharegpt": + phase = "sharegpt" + dataset = "sharegpt" + else: + continue + + # The last three numeric tokens are: concurrency, input_len/output_ctx, output_len. + try: + concurrency, input_len, output_len = int(parts[-3]), int(parts[-2]), int(parts[-1]) + except ValueError: continue - phase_num, concurrency, input_len, output_len = m.groups() - concurrency, input_len, output_len = int(concurrency), int(input_len), int(output_len) data = parse_jsonl(jsonl_path) if data is None: @@ -159,11 +170,11 @@ def main() -> None: scenario = { "name": scenario_name(concurrency, input_len, output_len), "config": { - "phase": f"phase{phase_num}", + "phase": phase, "concurrency": concurrency, "input_len": input_len, "output_len": output_len, - "dataset": "random", + "dataset": dataset, "num_prompts": metrics["success"] + metrics["failed"], }, "metrics": metrics, diff --git a/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh b/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh index 1f3e450..5c4614e 100755 --- a/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh +++ b/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh @@ -174,6 +174,60 @@ run_phase() { log "===== ${backend} ${phase} DONE =====" } +run_sharegpt() { + local backend="$1" + local phase="sharegpt" + local result_root="${RESULT_BASE}/${RUN_ID}/${backend}" + local raw_dir="${result_root}/raw_outputs" + local phase_log_dir="${result_root}/logs" + + mkdir -p "$raw_dir" "$phase_log_dir" + + local port + if [[ "$backend" == "sglang" ]]; then + port="$SGLANG_PORT" + else + port="$VLLM_PORT" + fi + + log "===== ${backend} ${phase} START (context_len=${SHAREGPT_CONTEXT_LEN}) =====" + + stop_server "$backend" + start_server "$backend" phase1 + run_warmup "$backend" phase1 + + for scenario in "${SHAREGPT_SCENARIOS[@]}"; do + read -r concurrency context_len output_len num_prompts <<< "$scenario" + output_file="${raw_dir}/${backend}_${phase}_$(date '+%m%d')_${concurrency}_${context_len}_${output_len}.jsonl" + detail_log="${phase_log_dir}/${backend}_${phase}_c${concurrency}_ctx${context_len}_o${output_len}.log" + + log "running ${backend} ${phase} scenario: c=${concurrency} ctx=${context_len} o=${output_len} n=${num_prompts}" + + "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \ + --backend "$backend" \ + --host 127.0.0.1 \ + --port "$port" \ + --dataset-name sharegpt \ + --dataset-path "$SHAREGPT_DATASET" \ + --sharegpt-output-len "$output_len" \ + --sharegpt-context-len "$context_len" \ + --num-prompts "$num_prompts" \ + --max-concurrency "$concurrency" \ + --request-rate 10000 \ + --output-file "$output_file" \ + --output-details \ + > "$detail_log" 2>&1 || { + log "ERROR: ${backend} ${phase} scenario c=${concurrency} ctx=${context_len} o=${output_len} failed; see ${detail_log}" + continue + } + + log "finished ${backend} ${phase} scenario: output=${output_file}" + done + + stop_server "$backend" + log "===== ${backend} ${phase} DONE =====" +} + parse_backend() { local backend="$1" local result_root="${RESULT_BASE}/${RUN_ID}/${backend}" @@ -245,11 +299,13 @@ write_backend_metadata vllm # Run SGLang. run_phase sglang phase1 +run_sharegpt sglang run_phase sglang phase2 parse_backend sglang # Run vLLM. run_phase vllm phase1 +run_sharegpt vllm run_phase vllm phase2 parse_backend vllm