diff --git a/experiments/dsv4_p800_max_context_length/README.md b/experiments/dsv4_p800_max_context_length/README.md new file mode 100644 index 0000000..45c2252 --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/README.md @@ -0,0 +1,59 @@ +# DSV4 P800 最大上下文长度探索实验 + +在 8x Kunlun P800 XPU 上,逐步增大输入长度,测试 SGLang serving `DeepSeek-V4-Flash-INT8` 时能稳定处理的最长上下文。 + +## 目的 + +- 找到 P800 + SGLang INT8 在 TP=8 配置下,单请求最多能支持多长的输入 token。 +- 每个长度只跑 1 个请求、输出 1 个 token,尽量减少 decode 阶段显存干扰。 +- 每次测试单独启动 server,并把 `context-length` 刚好设为 `target_len + CONTEXT_PAD`,避免预分配过大。 + +## 默认候选长度 + +```text +4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288 +``` + +可以通过环境变量覆盖: + +```bash +CANDIDATE_LENS=(4096 8192 16384) bash run_bench.sh +``` + +## 控制变量 + +| 维度 | 配置 | +|---|---| +| 硬件 | 8x Kunlun P800 XPU,TP=8 | +| 模型 | `DeepSeek-V4-Flash-INT8`(`w8a8_int8`) | +| 输入长度 | `--random-input-len $target_len --random-range-ratio 1.0` | +| 输出长度 | 1 | +| 并发 / 请求数 | 1 / 1 | +| 压测客户端 | 容器内 `sglang.bench_serving --backend sglang` | + +> `--random-range-ratio 1.0` 确保输入长度固定为 `target_len`,而不是默认的 `[1, target_len]` 均匀分布。 + +## 快速运行 + +```bash +bash experiments/dsv4_p800_max_context_length/run_bench.sh +``` + +结果保存在 `experiments/dsv4_p800_max_context_length/results//`: + +``` +results// +├── results.json +├── report.md +└── raw_outputs/ # gitignored +``` + +## 产物说明 + +- `results.json`:每个候选长度的尝试记录,包括 `success`、TTFT/TPOT/E2E 指标、`server_args`。 +- `report.md`:汇总表格,以及 SGLang INT8 最终支持的最大输入长度。 + +## 环境 + +- Docker image: `iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202` +- Container Python: `/root/miniconda/envs/python310_torch25_cuda/bin/python` diff --git a/experiments/dsv4_p800_max_context_length/config.env b/experiments/dsv4_p800_max_context_length/config.env new file mode 100644 index 0000000..8366da6 --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/config.env @@ -0,0 +1,43 @@ +# Common configuration for the max context length exploration on P800. +# All values can be overridden via environment variables. + +EXPERIMENT="dsv4_p800_max_context_length" +MODEL_NAME="DeepSeek-V4-Flash-INT8" +MODEL_PATH="/data1/models/DeepSeek-V4-Flash-INT8" +SERVED_MODEL_NAME="deepseek-v4-flash-int8" + +# Port must differ from other running experiments. +PORT="${PORT:-30013}" + +# We want exact input lengths, so fix the random range ratio to 1.0. +OUTPUT_LEN="${OUTPUT_LEN:-1}" +NUM_PROMPTS="${NUM_PROMPTS:-1}" +MAX_CONCURRENCY="${MAX_CONCURRENCY:-1}" +REQUEST_RATE="${REQUEST_RATE:-10000}" + +# Padding added to target length when setting server context length. +CONTEXT_PAD="${CONTEXT_PAD:-1024}" + +# Candidate input lengths (tokens). Override via CANDIDATE_LENS env var. +# Examples: +# CANDIDATE_LENS="4096 8192 16384" bash run_bench.sh +# CANDIDATE_LENS=(4096 8192 16384) bash run_bench.sh +if [[ -z "${CANDIDATE_LENS:-}" ]]; then + declare -a CANDIDATE_LENS=( + 4096 + 8192 + 16384 + 32768 + 65536 + 131072 + 262144 + 524288 + ) +elif ! declare -p CANDIDATE_LENS 2>/dev/null | grep -q '^declare -a'; then + # Normalize string-form arrays: remove surrounding parentheses and commas. + _CAND_STR="${CANDIDATE_LENS#(}" + _CAND_STR="${_CAND_STR%)}" + _CAND_STR="${_CAND_STR//,/}" + read -ra CANDIDATE_LENS <<< "$_CAND_STR" + unset _CAND_STR +fi diff --git a/experiments/dsv4_p800_max_context_length/extract_metrics.py b/experiments/dsv4_p800_max_context_length/extract_metrics.py new file mode 100755 index 0000000..c2579ce --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/extract_metrics.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Extract a compact metrics dict from a sglang.bench_serving JSONL output. + +Usage: + python3 extract_metrics.py +""" +import json +import sys +from pathlib import Path + + +def parse_jsonl(path: Path) -> dict | None: + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + return json.loads(line) + except json.JSONDecodeError: + continue + return None + + +def compute(data: dict) -> dict: + return { + "success_count": data.get("completed", 0), + "total_count": len(data.get("input_lens", [])), + "duration_s": data.get("duration", 0.0), + "request_throughput": data.get("request_throughput", 0.0), + "input_token_throughput": data.get("input_throughput", 0.0), + "output_token_throughput": data.get("output_throughput", 0.0), + "total_input_tokens": data.get("total_input_tokens", 0), + "total_output_tokens": data.get("total_output_tokens", 0), + "e2e_ms": { + "mean": data.get("mean_e2e_latency_ms", 0.0), + "p50": data.get("median_e2e_latency_ms", 0.0), + "p90": data.get("p90_e2e_latency_ms", 0.0), + "p95": data.get("p95_e2e_latency_ms", 0.0), + "p99": data.get("p99_e2e_latency_ms", 0.0), + }, + "ttft_ms": { + "mean": data.get("mean_ttft_ms", 0.0), + "p50": data.get("median_ttft_ms", 0.0), + "p90": data.get("p90_ttft_ms", 0.0), + "p95": data.get("p95_ttft_ms", 0.0), + "p99": data.get("p99_ttft_ms", 0.0), + }, + "tpot_ms": { + "mean": data.get("mean_tpot_ms", 0.0), + "p50": data.get("median_tpot_ms", 0.0), + "p90": data.get("p90_tpot_ms", 0.0), + "p95": data.get("p95_tpot_ms", 0.0), + "p99": data.get("p99_tpot_ms", 0.0), + }, + "itl_ms": { + "mean": data.get("mean_itl_ms", 0.0), + "p50": data.get("median_itl_ms", 0.0), + "p90": data.get("p90_itl_ms", 0.0), + "p95": data.get("p95_itl_ms", 0.0), + "p99": data.get("p99_itl_ms", 0.0), + }, + } + + +def main(): + path = Path(sys.argv[1]) + data = parse_jsonl(path) + if data is None: + raise SystemExit("No valid JSON found") + print(json.dumps(compute(data), indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_p800_max_context_length/parse_results.py b/experiments/dsv4_p800_max_context_length/parse_results.py new file mode 100755 index 0000000..715441f --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/parse_results.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Generate report.md from results.json for the max context length experiment. + +Usage: + python3 parse_results.py +""" +import json +import sys +from pathlib import Path + + +def main(): + result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") + results_json = result_root / "results.json" + report_path = result_root / "report.md" + + with open(results_json, "r", encoding="utf-8") as f: + data = json.load(f) + + meta = data["metadata"] + cfg = data["config"] + + with open(report_path, "w", encoding="utf-8") as f: + f.write("# P800 Max Context Length Exploration Report\n\n") + f.write(f"- Result root: `{result_root}`\n") + f.write(f"- Model: `{meta['model']}`\n") + f.write(f"- Hardware: {meta['hardware']}\n") + f.write(f"- TP: {cfg['tp']}, EP: {cfg['ep']}\n") + f.write(f"- Output len: {cfg['output_len']}, num prompts: {cfg['num_prompts']}, concurrency: {cfg['max_concurrency']}\n\n") + + f.write("## Per-attempt results\n\n") + f.write("| Target len | Max context len | Status | TTFT mean(ms) | TTFT P95(ms) | E2E mean(ms) | E2E P99(ms) | Error |\n") + f.write("|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + max_supported = 0 + for s in data.get("scenarios", []): + target = s["target_len"] + status = "✅ pass" if s["success"] else "❌ fail" + metrics = s.get("metrics") or {} + ttft = metrics.get("ttft_ms", {}) + e2e = metrics.get("e2e_ms", {}) + f.write( + f"| {target} | {s['max_context_len']} | {status} | " + f"{ttft.get('mean', 0):.2f} | {ttft.get('p95', 0):.2f} | " + f"{e2e.get('mean', 0):.2f} | {e2e.get('p99', 0):.2f} | " + f"{s.get('error', '') or ''} |\n" + ) + if s["success"] and target > max_supported: + max_supported = target + + f.write("\n## Maximum supported input length\n\n") + f.write(f"- **sglang-xpu**: {max_supported} tokens\n\n") + + print(f"Wrote report to {report_path}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_p800_max_context_length/results/20260709-032422/report.md b/experiments/dsv4_p800_max_context_length/results/20260709-032422/report.md new file mode 100644 index 0000000..d2f0b79 --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/results/20260709-032422/report.md @@ -0,0 +1,18 @@ +# P800 Max Context Length Exploration Report + +- Result root: `/data1/yy/sskj/experiments/dsv4_p800_max_context_length/results/20260709-032422` +- Model: `/data1/models/DeepSeek-V4-Flash-INT8` +- Hardware: 8x Kunlun P800 XPU +- TP: 8, EP: 8 +- Output len: 1, num prompts: 1, concurrency: 1 + +## Per-attempt results + +| Target len | Max context len | Status | TTFT mean(ms) | TTFT P95(ms) | E2E mean(ms) | E2E P99(ms) | Error | +|---:|---:|---:|---:|---:|---:|---:|---:| +| 4096 | 5120 | ✅ pass | 231.33 | 0.00 | 231.34 | 231.34 | | + +## Maximum supported input length + +- **sglang-xpu**: 4096 tokens + diff --git a/experiments/dsv4_p800_max_context_length/results/20260709-032422/results.json b/experiments/dsv4_p800_max_context_length/results/20260709-032422/results.json new file mode 100644 index 0000000..60262e2 --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/results/20260709-032422/results.json @@ -0,0 +1,76 @@ +{ + "metadata": { + "experiment": "dsv4_p800_max_context_length", + "run_id": "20260709-032422", + "timestamp": "2026-07-09T03:24:22+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_max_context_length/run_bench.sh", + "env": "", + "git_commit": "1d1c79d", + "git_dirty": "dirty", + "description": "P800 max context length exploration for DeepSeek-V4-Flash-INT8" + }, + "config": { + "tp": 8, + "ep": 8, + "xpu_visible_devices": "0,1,2,3,4,5,6,7", + "output_len": 1, + "num_prompts": 1, + "max_concurrency": 1, + "request_rate": 10000, + "context_pad": 1024 + }, + "scenarios": [ + { + "backend": "sglang", + "target_len": 4096, + "max_context_len": 5120, + "success": true, + "server_args": "python -m sglang.launch_server --host 0.0.0.0 --port 30013 --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens 5120 --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 5120", + "metrics": { + "success_count": 1, + "total_count": 1, + "duration_s": 3.3442696970305406, + "request_throughput": 0.2990189460162034, + "input_token_throughput": 1224.781602882369, + "output_token_throughput": 0.2990189460162034, + "total_input_tokens": 4096, + "total_output_tokens": 1, + "e2e_ms": { + "mean": 231.34461202425882, + "p50": 231.34461202425882, + "p90": 231.34461202425882, + "p95": 0.0, + "p99": 231.34461202425882 + }, + "ttft_ms": { + "mean": 231.33470199536532, + "p50": 231.33470199536532, + "p90": 0.0, + "p95": 0.0, + "p99": 231.33470199536532 + }, + "tpot_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + }, + "itl_ms": { + "mean": 0.0, + "p50": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0 + } + }, + "error": null + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_p800_max_context_length/run_bench.sh b/experiments/dsv4_p800_max_context_length/run_bench.sh new file mode 100755 index 0000000..2b5958d --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/run_bench.sh @@ -0,0 +1,245 @@ +#!/usr/bin/env bash +# Max context length exploration for P800 SGLang INT8. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" +LOG_DIR="${RESULT_ROOT}/logs" +RAW_DIR="${RESULT_ROOT}/raw_outputs" +CONTAINER_NAME="${CONTAINER_NAME:-sglang-dsv4-flash}" +CONTAINER_PYTHON="${CONTAINER_PYTHON:-/root/miniconda/envs/python310_torch25_cuda/bin/python}" + +ensure_result_root "$RESULT_ROOT" +log_init "${LOG_DIR}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "result_root=${RESULT_ROOT}" +log "platform=${PLATFORM}" +log "hardware=${HARDWARE}" +log "model=${MODEL_PATH}" +log "candidate_lengths=${CANDIDATE_LENS[*]}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +is_server_healthy() { + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1 +} + +stop_server() { + log "stopping container ${CONTAINER_NAME}" + docker rm -f "$CONTAINER_NAME" 2>/dev/null || true + pkill -9 -f 'sglang.launch_server' 2>/dev/null || true + sleep 2 +} + +start_server() { + local target_len="$1" + log "starting server (target_len=${target_len})" + bash "${SCRIPT_DIR}/start_server.sh" "$target_len" >> "${LOG_DIR}/start_server_${target_len}.log" 2>&1 +} + +build_server_args() { + local target_len="$1" + local max_context_len=$(( target_len + CONTEXT_PAD )) + echo "python -m sglang.launch_server --host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens ${max_context_len} --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${max_context_len}" +} + +append_scenario() { + local scenario_json="$1" + python3 - "$RESULT_JSON" "$scenario_json" <<'PY' +import json +import sys + +results_path, scenario_json = sys.argv[1], sys.argv[2] +with open(results_path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["scenarios"].append(json.loads(scenario_json)) +with open(results_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY +} + +# Build a scenario JSON blob safely with Python. +build_scenario_json() { + local target_len="$1" + local max_context_len="$2" + local success="$3" + local server_args="$4" + local error_msg="$5" + local metrics_json="${6:-null}" + + python3 - "$target_len" "$max_context_len" "$success" "$server_args" "$error_msg" "$metrics_json" <<'PY' +import json +import sys + +target_len, max_context_len, success, server_args, error_msg, metrics_json = sys.argv[1:7] +metrics = json.loads(metrics_json) if metrics_json not in ("null", "") else None +print(json.dumps({ + "backend": "sglang", + "target_len": int(target_len), + "max_context_len": int(max_context_len), + "success": success == "true", + "server_args": server_args, + "metrics": metrics, + "error": error_msg or None, +}, ensure_ascii=False)) +PY +} + +run_length() { + local target_len="$1" + local max_context_len=$(( target_len + CONTEXT_PAD )) + local output_file="${RAW_DIR}/sglang_ctx_${target_len}.jsonl" + local container_output="/tmp/bench_outputs/sglang_ctx_${target_len}.jsonl" + local detail_log="${LOG_DIR}/sglang_ctx_${target_len}.log" + + log "trying sglang target_len=${target_len} (context-length=${max_context_len})" + + stop_server + local server_args + server_args="$(build_server_args "$target_len")" + if ! start_server "$target_len"; then + local err="server failed to start" + log "ERROR: ${err}" + append_scenario "$(build_scenario_json "$target_len" "$max_context_len" "false" "$server_args" "$err")" + return 1 + fi + + docker exec "$CONTAINER_NAME" mkdir -p "$(dirname "$container_output")" + + local bench_rc=0 + docker exec "$CONTAINER_NAME" \ + env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \ + "${CONTAINER_PYTHON}" -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "$PORT" \ + --model "$MODEL_PATH" \ + --dataset-name random \ + --dataset-path /workspace/dummy_sharegpt.json \ + --random-input-len "$target_len" \ + --random-output-len "$OUTPUT_LEN" \ + --random-range-ratio 1.0 \ + --num-prompts "$NUM_PROMPTS" \ + --max-concurrency "$MAX_CONCURRENCY" \ + --request-rate "$REQUEST_RATE" \ + --output-file "$container_output" \ + --output-details \ + > "$detail_log" 2>&1 || bench_rc=$? + + if [[ "$bench_rc" -ne 0 ]]; then + local err="bench_serving exited with code ${bench_rc}; see ${detail_log}" + log "ERROR: sglang target_len=${target_len} failed; ${err}" + append_scenario "$(build_scenario_json "$target_len" "$max_context_len" "false" "$server_args" "$err")" + stop_server + return 1 + fi + + docker cp "${CONTAINER_NAME}:${container_output}" "$output_file" || { + local err="failed to copy output from container" + log "ERROR: ${err}" + append_scenario "$(build_scenario_json "$target_len" "$max_context_len" "false" "$server_args" "$err")" + stop_server + return 1 + } + + # Verify the request actually completed. + local completed + completed="$(python3 -c " +import json +with open('${output_file}') as f: + for line in f: + data = json.loads(line) + print(data.get('completed', 0)) + break +")" + if [[ "${completed:-0}" -lt "$NUM_PROMPTS" ]]; then + local err="only ${completed}/${NUM_PROMPTS} requests completed" + log "ERROR: sglang target_len=${target_len} ${err}" + append_scenario "$(build_scenario_json "$target_len" "$max_context_len" "false" "$server_args" "$err")" + stop_server + return 1 + fi + + local metrics_json + metrics_json="$(python3 "${SCRIPT_DIR}/extract_metrics.py" "$output_file")" + + append_scenario "$(build_scenario_json "$target_len" "$max_context_len" "true" "$server_args" "" "$metrics_json")" + + log "sglang target_len=${target_len} succeeded" + stop_server +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +mkdir -p "$RAW_DIR" "$LOG_DIR" + +RESULT_JSON="${RESULT_ROOT}/results.json" +write_metadata_json \ + "$RESULT_JSON" \ + "$EXPERIMENT_NAME" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "sglang" \ + "sglang-xpu" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "" \ + "P800 max context length exploration for DeepSeek-V4-Flash-INT8" + +# Embed config. +python3 - "$RESULT_JSON" <<'PY' +import json +import sys + +path = sys.argv[1] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["config"] = { + "tp": 8, + "ep": 8, + "xpu_visible_devices": "0,1,2,3,4,5,6,7", + "output_len": 1, + "num_prompts": 1, + "max_concurrency": 1, + "request_rate": 10000, + "context_pad": 1024, +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY + +stop_server + +for target_len in "${CANDIDATE_LENS[@]}"; do + if ! run_length "$target_len"; then + log "stopping exploration after first failure at target_len=${target_len}" + break + fi +done + +log "parsing results" +python3 "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || { + log "WARNING: parse_results.py failed; see ${LOG_DIR}/parse.log" +} + +log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_p800_max_context_length/start_server.sh b/experiments/dsv4_p800_max_context_length/start_server.sh new file mode 100755 index 0000000..0dbb973 --- /dev/null +++ b/experiments/dsv4_p800_max_context_length/start_server.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Start P800 SGLang INT8 server for a specific target context length. +set -Eeuo pipefail + +TARGET_LEN="${1:-}" +if [[ -z "$TARGET_LEN" ]]; then + echo "Usage: $0 " + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +MAX_CONTEXT_LEN=$(( TARGET_LEN + CONTEXT_PAD )) +CONTAINER_NAME="${CONTAINER_NAME:-sglang-dsv4-flash}" +DOCKER_IMAGE="${DOCKER_IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202}" +PATCH_ROOT="${PATCH_ROOT:-${ROOT_DIR}/platforms/patches/kunlun_p800}" +RESULT_ROOT="${RESULT_ROOT:-/tmp/${EXPERIMENT}}" +SERVER_LOG="${RESULT_ROOT}/logs/server_${TARGET_LEN}.outer.log" + +mkdir -p "$(dirname "$SERVER_LOG")" + +log "starting P800 SGLang INT8 server (target_len=${TARGET_LEN}, context-length=${MAX_CONTEXT_LEN})" +log "model: ${MODEL_PATH}" +log "port: ${PORT}" +log "server log: ${SERVER_LOG}" + +# Stop any existing container with the same name. +docker rm -f "$CONTAINER_NAME" 2>/dev/null || true + +# Build device args. +device_args="" +for i in 0 1 2 3 4 5 6 7; do + device_args="${device_args} --device /dev/xpu${i}:/dev/xpu${i}" +done +device_args="${device_args} --device /dev/xpuctrl:/dev/xpuctrl" + +# Environment variables required by the P800 SGLang INT8 image. +env_args=( + -e XPU_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + -e CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + -e CUDA_DEVICE_ORDER=OAM_ID + -e SGLANG_USE_TRANSFORMERS_V5_TOKENIZER=1 + -e XMLIR_FORCE_USE_XPU_GRAPH=1 + -e SGLANG_DSV4_MODE=2604 + -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + -e SGLANG_NSA_DUAL_STREAM=true + -e SGLANG_NSA_QUANT_WQ_B_WK=false + -e SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 + -e SGLANG_OPT_DEEPGEMM_HC_PRENORM=false + -e SGLANG_OPT_USE_TILELANG_MHC_PRE=1 + -e SGLANG_OPT_USE_TILELANG_MHC_POST=1 + -e SGLANG_CLEAN_REQUEST_WHEN_RETRACT=1 + -e SGLANG_SET_CPU_AFFINITY=1 + -e SGLANG_OPT_USE_KLX_TOPK_KERNEL=1 + -e XSGL_INTERTYPE_BFP16=1 + -e ENABLE_FAST_BFP16_ATTN=1 + -e XSGL_USE_DEEP_GEMM_BMM=1 + -e XSGL_XDNN_QUANT=1 + -e XSGL_FUSE_RMS_NORM_QUANT=1 + -e XSGL_TRANSPOSE_MATMUL_WEIGHT=1 + -e XINFER_QUANT_SDNN=1 + -e XSGL_USE_MOE_SIGMOID_GROUP_TOPK_NORM=1 + -e XSGL_EARLY_FIRST_TOKEN=1 + -e XSGL_ENABLE_TGEMM_FP16=1 + -e SGLANG_ENABLE_SPEC_V2=True + -e SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + -e PYTHONDONTWRITEBYTECODE=1 + -e XTORCH_OPS_LIB_DIR=/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/xtorch_ops + -e XPU_RUNTIME_LIB_DIR=/root/miniconda/envs/python310_torch25_cuda/xcudart/lib + -e BKCL_TREE_THRESHOLD=1048576 + -e CUDA_ENABLE_P2P_NO_UVA=1 + -e NCCL_IB_GID_INDEX=3 + -e IS_DSV4=1 + -e MC_CUSTOM_TOPO_JSON=/workspace/nic_priority_matrix_test.json + # INT8 specific + -e SGLANG_DSV4_FP4_EXPERTS=false + -e SGLANG_APPLY_CONFIG_BACKUP=auto + -e BKCL_ENABLE_XDR=1 + -e BKCL_RDMA_NICS=eth1,eth1,eth3,eth3,eth5,eth5,eth7,eth7 + -e BKCL_RDMA_VERBS=1 + -e XSGL_INT8_LM_HEAD=1 + -e SGLANG_P800_ALL_GATHER_FALLBACK=0 +) + +# Launch args. Based on the proven P800 INT8 command from server_docker.sh, +# but with dynamic --context-length and --max-prefill-tokens for the max-context +# test. Keep --max-running-requests at 64 because setting it to 1 causes +# req_to_token_pool allocation failures in this image. +launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.8 --max-prefill-tokens ${MAX_CONTEXT_LEN} --max-running-requests 64 --tensor-parallel-size 8 --ep-size 8 --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${MAX_CONTEXT_LEN}" + +# Base64-encode the bootstrap command to avoid host-shell quoting issues. +server_cmd=$(cat </dev/null || true +/root/miniconda/envs/python310_torch25_cuda/bin/pip install --upgrade safetensors -q +/root/miniconda/envs/python310_torch25_cuda/bin/pip install https://files.pythonhosted.org/packages/14/8b/2a1333a6455c6fad401c2285dee6f58016c55b1cb44cae3a31f8a9cc7d83/apache_tvm_ffi-0.1.0b2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl -q +/root/miniconda/envs/python310_torch25_cuda/bin/python -c "import torch; torch.float8_e8m0fnu = torch.uint8; import runpy, sys; sys.argv[0] = 'sglang.launch_server'; runpy.run_module('sglang.launch_server', run_name='__main__')" ${launch_args} +EOF +) +server_cmd_b64=$(printf '%s' "$server_cmd" | base64 -w0) + +patch_mounts=( + -v "${PATCH_ROOT}/nic_priority_matrix_test.json:/workspace/nic_priority_matrix_test.json:ro" + -v "${PATCH_ROOT}/dummy_sharegpt.json:/workspace/dummy_sharegpt.json:ro" +) + +docker run -d \ + --name "${CONTAINER_NAME}" \ + --privileged \ + --network host \ + --ipc host \ + ${device_args} \ + -v "${MODEL_PATH}:/models:ro" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + "${patch_mounts[@]}" \ + "${env_args[@]}" \ + "${DOCKER_IMAGE}" \ + bash -c "echo '${server_cmd_b64}' | base64 -d | bash" \ + >> "${SERVER_LOG}" 2>&1 + +log "container ${CONTAINER_NAME} started, waiting for health" +if health_check 127.0.0.1 "$PORT" 600; then + log "container ${CONTAINER_NAME} is healthy" +else + log "ERROR: container ${CONTAINER_NAME} failed health check" + exit 1 +fi diff --git a/experiments/dsv4_p800_sglang/results/20260708-043404/results.json b/experiments/dsv4_p800_sglang/results/20260708-043404/results.json new file mode 100644 index 0000000..79f632f --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-043404/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-043404", + "timestamp": "2026-07-08T04:34:04+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} diff --git a/experiments/dsv4_p800_sglang/results/20260708-043830/results.json b/experiments/dsv4_p800_sglang/results/20260708-043830/results.json new file mode 100644 index 0000000..5335eef --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-043830/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-043830", + "timestamp": "2026-07-08T04:38:30+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} diff --git a/experiments/dsv4_p800_sglang/results/20260708-044535/results.json b/experiments/dsv4_p800_sglang/results/20260708-044535/results.json new file mode 100644 index 0000000..94df31b --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-044535/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-044535", + "timestamp": "2026-07-08T04:45:35+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} diff --git a/experiments/dsv4_p800_sglang/results/20260708-045327/results.json b/experiments/dsv4_p800_sglang/results/20260708-045327/results.json new file mode 100644 index 0000000..f2b1bbe --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-045327/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-045327", + "timestamp": "2026-07-08T04:53:27+00:00", + "model": "/data1/models/deepseek-ai/DeepSeek-V4-Flash", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} diff --git a/experiments/dsv4_p800_sglang/results/20260708-045554/results.json b/experiments/dsv4_p800_sglang/results/20260708-045554/results.json new file mode 100644 index 0000000..a95c6d2 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-045554/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-045554", + "timestamp": "2026-07-08T04:55:54+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} diff --git a/experiments/dsv4_p800_sglang/results/20260708-045616/results.json b/experiments/dsv4_p800_sglang/results/20260708-045616/results.json new file mode 100644 index 0000000..70013bf --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-045616/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-045616", + "timestamp": "2026-07-08T04:56:16+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} diff --git a/experiments/dsv4_p800_sglang/results/20260708-050025/results.json b/experiments/dsv4_p800_sglang/results/20260708-050025/results.json new file mode 100644 index 0000000..5ec902a --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-050025/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-050025", + "timestamp": "2026-07-08T05:00:25+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} diff --git a/experiments/dsv4_p800_sglang/results/20260708-050713/report.md b/experiments/dsv4_p800_sglang/results/20260708-050713/report.md new file mode 100644 index 0000000..fe11a79 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-050713/report.md @@ -0,0 +1,16 @@ +# Benchmark Report: dsv4_p800_sglang + +## Metadata + +- **Run ID**: 20260708-050713 +- **Timestamp**: 2026-07-08T05:07:13+00:00 +- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 +- **Engine/Backend**: sglang-xpu / sglang +- **Hardware**: 8x Kunlun P800 XPU +- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 +- **Git Commit**: 2c332ff + +## Results + +| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| diff --git a/experiments/dsv4_p800_sglang/results/20260708-050713/results.json b/experiments/dsv4_p800_sglang/results/20260708-050713/results.json new file mode 100644 index 0000000..06ec92b --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-050713/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-050713", + "timestamp": "2026-07-08T05:07:13+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-050722/report.md b/experiments/dsv4_p800_sglang/results/20260708-050722/report.md new file mode 100644 index 0000000..9d497fe --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-050722/report.md @@ -0,0 +1,16 @@ +# Benchmark Report: dsv4_p800_sglang + +## Metadata + +- **Run ID**: 20260708-050722 +- **Timestamp**: 2026-07-08T05:07:22+00:00 +- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 +- **Engine/Backend**: sglang-xpu / sglang +- **Hardware**: 8x Kunlun P800 XPU +- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 +- **Git Commit**: 2c332ff + +## Results + +| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| diff --git a/experiments/dsv4_p800_sglang/results/20260708-050722/results.json b/experiments/dsv4_p800_sglang/results/20260708-050722/results.json new file mode 100644 index 0000000..75d0b0f --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-050722/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-050722", + "timestamp": "2026-07-08T05:07:22+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-050858/report.md b/experiments/dsv4_p800_sglang/results/20260708-050858/report.md new file mode 100644 index 0000000..7bfb506 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-050858/report.md @@ -0,0 +1,16 @@ +# Benchmark Report: dsv4_p800_sglang + +## Metadata + +- **Run ID**: 20260708-050858 +- **Timestamp**: 2026-07-08T05:08:58+00:00 +- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 +- **Engine/Backend**: sglang-xpu / sglang +- **Hardware**: 8x Kunlun P800 XPU +- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 +- **Git Commit**: 2c332ff + +## Results + +| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| diff --git a/experiments/dsv4_p800_sglang/results/20260708-050858/results.json b/experiments/dsv4_p800_sglang/results/20260708-050858/results.json new file mode 100644 index 0000000..7978092 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-050858/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-050858", + "timestamp": "2026-07-08T05:08:58+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-051023/report.md b/experiments/dsv4_p800_sglang/results/20260708-051023/report.md new file mode 100644 index 0000000..4baebbb --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-051023/report.md @@ -0,0 +1,16 @@ +# Benchmark Report: dsv4_p800_sglang + +## Metadata + +- **Run ID**: 20260708-051023 +- **Timestamp**: 2026-07-08T05:10:23+00:00 +- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 +- **Engine/Backend**: sglang-xpu / sglang +- **Hardware**: 8x Kunlun P800 XPU +- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 +- **Git Commit**: 2c332ff + +## Results + +| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| diff --git a/experiments/dsv4_p800_sglang/results/20260708-051023/results.json b/experiments/dsv4_p800_sglang/results/20260708-051023/results.json new file mode 100644 index 0000000..e0330b9 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-051023/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-051023", + "timestamp": "2026-07-08T05:10:23+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-051200/results.json b/experiments/dsv4_p800_sglang/results/20260708-051200/results.json new file mode 100644 index 0000000..b806584 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-051200/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-051200", + "timestamp": "2026-07-08T05:12:00+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} diff --git a/experiments/dsv4_p800_sglang/results/20260708-051316/report.md b/experiments/dsv4_p800_sglang/results/20260708-051316/report.md new file mode 100644 index 0000000..4a72f6c --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-051316/report.md @@ -0,0 +1,17 @@ +# Benchmark Report: dsv4_p800_sglang + +## Metadata + +- **Run ID**: 20260708-051316 +- **Timestamp**: 2026-07-08T05:13:16+00:00 +- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 +- **Engine/Backend**: sglang-xpu / sglang +- **Hardware**: 8x Kunlun P800 XPU +- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 +- **Git Commit**: 2c332ff + +## Results + +| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c32_i512_o256 | 32 | 512/256 | 10 | 0 | 1.87 | 262.65 | 506.4399079710711 | 507.2712521097855 | 21.948128755653826 | 29.154062995221466 | 5314.590020017349 | diff --git a/experiments/dsv4_p800_sglang/results/20260708-051316/results.json b/experiments/dsv4_p800_sglang/results/20260708-051316/results.json new file mode 100644 index 0000000..60b7466 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-051316/results.json @@ -0,0 +1,2482 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-051316", + "timestamp": "2026-07-08T05:13:16+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "2c332ff", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [ + { + "name": "c32_i512_o256", + "concurrency": 32, + "input_len": 512, + "output_len": 256, + "success": 10, + "failed": 0, + "duration_s": 5.35, + "request_throughput": 1.87, + "input_token_throughput": 470.87, + "output_token_throughput": 262.65, + "total_token_throughput": 733.53, + "accept_length": null, + "latencies": { + "e2e_ms": { + "mean": 3372.781996097183, + "p50": 3498.675397480838, + "p90": 5125.400093087228, + "p95": null, + "p99": 5314.590020017349 + }, + "ttft_ms": { + "mean": 478.8548628857825, + "p50": 506.4399079710711, + "p90": null, + "p95": null, + "p99": 507.2712521097855 + }, + "tpot_ms": { + "mean": 22.334078483797782, + "p50": 21.948128755653826, + "p90": null, + "p95": null, + "p99": 29.154062995221466 + }, + "itl_ms": { + "mean": 20.75981864563044, + "p50": 18.560815330905218, + "p90": null, + "p95": 30.416983005125076, + "p99": 76.09861597185954 + } + }, + "raw_requests": [ + { + "tag": null, + "backend": "sglang", + "dataset_name": "random", + "request_rate": Infinity, + "max_concurrency": 32, + "sharegpt_output_len": null, + "random_input_len": 512, + "random_output_len": 256, + "random_range_ratio": 0.0, + "server_info": { + "model_path": "/data1/models/DeepSeek-V4-Flash-INT8", + "tokenizer_path": "/data1/models/DeepSeek-V4-Flash-INT8", + "tokenizer_mode": "auto", + "tokenizer_worker_num": 1, + "skip_tokenizer_init": false, + "load_format": "auto", + "model_loader_extra_config": "{}", + "trust_remote_code": true, + "context_length": null, + "is_embedding": false, + "enable_multimodal": null, + "revision": null, + "model_impl": "auto", + "host": "0.0.0.0", + "port": 30000, + "fastapi_root_path": "", + "grpc_mode": false, + "skip_server_warmup": false, + "warmups": null, + "nccl_port": null, + "checkpoint_engine_wait_weights_before_ready": false, + "ssl_keyfile": null, + "ssl_certfile": null, + "ssl_ca_certs": null, + "ssl_keyfile_password": null, + "enable_ssl_refresh": false, + "dtype": "auto", + "quantization": "w8a8_int8", + "quantization_param_path": null, + "kv_cache_dtype": "float16", + "enable_fp32_lm_head": false, + "modelopt_quant": null, + "modelopt_checkpoint_restore_path": null, + "modelopt_checkpoint_save_path": null, + "modelopt_export_path": null, + "quantize_and_serve": false, + "rl_quant_profile": null, + "mem_fraction_static": 0.8, + "max_running_requests": 64, + "max_queued_requests": null, + "max_total_tokens": null, + "chunked_prefill_size": 8192, + "enable_dynamic_chunking": false, + "max_prefill_tokens": 16384, + "prefill_max_requests": null, + "custom_max_context_len": null, + "schedule_policy": "fcfs", + "enable_priority_scheduling": false, + "disable_priority_preemption": false, + "default_priority_value": null, + "abort_on_priority_when_disabled": false, + "schedule_low_priority_values_first": false, + "priority_scheduling_preemption_threshold": 10, + "schedule_conservativeness": 1.0, + "page_size": 256, + "swa_full_tokens_ratio": 0.1, + "disable_hybrid_swa_memory": false, + "radix_eviction_policy": "lru", + "enable_prefill_delayer": false, + "prefill_delayer_max_delay_passes": 30, + "prefill_delayer_token_usage_low_watermark": null, + "prefill_delayer_forward_passes_buckets": null, + "prefill_delayer_wait_seconds_buckets": null, + "device": "cuda", + "tp_size": 8, + "pp_size": 1, + "pp_max_micro_batch_size": null, + "pp_async_batch_depth": 0, + "stream_interval": 1, + "stream_response_default_include_usage": false, + "incremental_streaming_output": false, + "enable_streaming_session": false, + "random_seed": 228424360, + "constrained_json_whitespace_pattern": null, + "constrained_json_disable_any_whitespace": true, + "watchdog_timeout": 3000000.0, + "soft_watchdog_timeout": null, + "dist_timeout": null, + "download_dir": null, + "model_checksum": null, + "base_gpu_id": 0, + "gpu_id_step": 1, + "sleep_on_idle": false, + "use_ray": false, + "custom_sigquit_handler": null, + "log_level": "info", + "log_level_http": null, + "log_requests": false, + "log_requests_level": 2, + "log_requests_format": "text", + "log_requests_target": null, + "uvicorn_access_log_exclude_prefixes": [], + "crash_dump_folder": null, + "show_time_cost": false, + "enable_metrics": true, + "enable_mfu_metrics": false, + "enable_metrics_for_all_schedulers": false, + "tokenizer_metrics_custom_labels_header": "x-custom-labels", + "tokenizer_metrics_allowed_custom_labels": null, + "extra_metric_labels": null, + "bucket_time_to_first_token": null, + "bucket_inter_token_latency": null, + "bucket_e2e_request_latency": null, + "collect_tokens_histogram": false, + "prompt_tokens_buckets": null, + "generation_tokens_buckets": null, + "gc_warning_threshold_secs": 0.0, + "decode_log_interval": 40, + "enable_request_time_stats_logging": true, + "kv_events_config": null, + "enable_trace": false, + "otlp_traces_endpoint": "localhost:4317", + "export_metrics_to_file": false, + "export_metrics_to_file_dir": null, + "api_key": null, + "admin_api_key": null, + "served_model_name": "/data1/models/DeepSeek-V4-Flash-INT8", + "weight_version": "default", + "chat_template": null, + "hf_chat_template_name": null, + "completion_template": null, + "file_storage_path": "sglang_storage", + "enable_cache_report": false, + "reasoning_parser": "deepseek-v4", + "tool_call_parser": "deepseekv4", + "tool_server": null, + "sampling_defaults": "model", + "dp_size": 1, + "load_balance_method": "round_robin", + "attn_cp_size": 1, + "moe_dp_size": 1, + "dist_init_addr": null, + "nnodes": 1, + "node_rank": 0, + "json_model_override_args": "{}", + "preferred_sampling_params": null, + "enable_lora": null, + "enable_lora_overlap_loading": null, + "max_lora_rank": null, + "lora_target_modules": null, + "lora_paths": null, + "max_loaded_loras": null, + "max_loras_per_batch": 8, + "lora_eviction_policy": "lru", + "lora_backend": "csgmv", + "max_lora_chunk_size": 16, + "experts_shared_outer_loras": null, + "attention_backend": "compressed", + "decode_attention_backend": null, + "prefill_attention_backend": null, + "sampling_backend": "flashinfer", + "grammar_backend": "xgrammar", + "mm_attention_backend": null, + "fp8_gemm_runner_backend": "auto", + "fp4_gemm_runner_backend": "auto", + "nsa_prefill_backend": "klxdsa", + "nsa_decode_backend": "klxdsa", + "disable_flashinfer_autotune": false, + "mamba_backend": "triton", + "speculative_algorithm": "EAGLE", + "speculative_draft_model_path": "/data1/models/DeepSeek-V4-Flash-INT8", + "speculative_draft_model_revision": null, + "speculative_draft_load_format": null, + "speculative_num_steps": 3, + "speculative_eagle_topk": 1, + "speculative_num_draft_tokens": 4, + "speculative_accept_threshold_single": 1.0, + "speculative_accept_threshold_acc": 1.0, + "speculative_token_map": null, + "speculative_attention_mode": "prefill", + "speculative_draft_attention_backend": null, + "speculative_moe_runner_backend": "auto", + "speculative_moe_a2a_backend": null, + "speculative_draft_model_quantization": "w8a8_int8", + "speculative_ngram_min_bfs_breadth": 1, + "speculative_ngram_max_bfs_breadth": 10, + "speculative_ngram_match_type": "BFS", + "speculative_ngram_max_trie_depth": 18, + "speculative_ngram_capacity": 10000000, + "enable_multi_layer_eagle": false, + "ep_size": 8, + "moe_a2a_backend": "none", + "moe_runner_backend": "auto", + "flashinfer_mxfp4_moe_precision": "default", + "enable_hisparse": false, + "enable_flashinfer_allreduce_fusion": false, + "enforce_disable_flashinfer_allreduce_fusion": false, + "enable_aiter_allreduce_fusion": false, + "deepep_mode": "auto", + "ep_num_redundant_experts": 0, + "ep_dispatch_algorithm": null, + "init_expert_location": "trivial", + "enable_eplb": false, + "eplb_algorithm": "auto", + "eplb_rebalance_num_iterations": 1000, + "eplb_rebalance_layers_per_chunk": null, + "eplb_min_rebalancing_utilization_threshold": 1.0, + "expert_distribution_recorder_mode": null, + "expert_distribution_recorder_buffer_size": 1000, + "enable_expert_distribution_metrics": false, + "deepep_config": null, + "moe_dense_tp_size": null, + "elastic_ep_backend": null, + "enable_elastic_expert_backup": false, + "mooncake_ib_device": null, + "max_mamba_cache_size": null, + "mamba_ssm_dtype": null, + "mamba_full_memory_ratio": 0.9, + "mamba_scheduler_strategy": "no_buffer", + "mamba_track_interval": 256, + "linear_attn_backend": "triton", + "linear_attn_decode_backend": null, + "linear_attn_prefill_backend": null, + "enable_hierarchical_cache": false, + "hicache_ratio": 2.0, + "hicache_size": 0, + "hicache_write_policy": "write_through", + "hicache_io_backend": "kernel", + "hicache_mem_layout": "page_first", + "hicache_storage_backend": null, + "hicache_storage_prefetch_policy": "best_effort", + "hicache_storage_backend_extra_config": null, + "hisparse_config": null, + "enable_lmcache": false, + "kt_weight_path": null, + "kt_method": "AMXINT4", + "kt_cpuinfer": null, + "kt_threadpool_count": 2, + "kt_num_gpu_experts": null, + "kt_max_deferred_experts_per_token": null, + "dllm_algorithm": null, + "dllm_algorithm_config": null, + "enable_double_sparsity": false, + "ds_channel_config_path": null, + "ds_heavy_channel_num": 32, + "ds_heavy_token_num": 256, + "ds_heavy_channel_type": "qk", + "ds_sparse_decode_threshold": 4096, + "cpu_offload_gb": 0, + "offload_group_size": -1, + "offload_num_in_group": 1, + "offload_prefetch_step": 1, + "offload_mode": "cpu", + "multi_item_scoring_delimiter": null, + "disable_radix_cache": false, + "cuda_graph_max_bs": 32, + "cuda_graph_bs": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 10, + 12, + 14, + 16, + 18, + 20, + 22, + 24, + 26, + 28, + 30, + 32 + ], + "disable_cuda_graph": false, + "disable_cuda_graph_padding": false, + "enable_profile_cuda_graph": false, + "enable_cudagraph_gc": false, + "enable_layerwise_nvtx_marker": false, + "enable_nccl_nvls": false, + "enable_symm_mem": false, + "disable_flashinfer_cutlass_moe_fp4_allgather": false, + "enable_tokenizer_batch_encode": false, + "disable_tokenizer_batch_decode": false, + "disable_outlines_disk_cache": false, + "disable_custom_all_reduce": true, + "enable_mscclpp": false, + "enable_torch_symm_mem": false, + "pre_warm_nccl": false, + "disable_overlap_schedule": false, + "enable_mixed_chunk": false, + "enable_dp_attention": false, + "enable_dp_lm_head": false, + "enable_two_batch_overlap": false, + "enable_single_batch_overlap": false, + "tbo_token_distribution_threshold": 0.48, + "enable_torch_compile": false, + "disable_piecewise_cuda_graph": true, + "enforce_piecewise_cuda_graph": false, + "enable_torch_compile_debug_mode": false, + "torch_compile_max_bs": 32, + "piecewise_cuda_graph_max_tokens": 8192, + "piecewise_cuda_graph_tokens": [ + 4, + 8, + 12, + 16, + 20, + 24, + 28, + 32, + 48, + 64, + 80, + 96, + 112, + 128, + 144, + 160, + 176, + 192, + 208, + 224, + 240, + 256, + 288, + 320, + 352, + 384, + 416, + 448, + 480, + 512, + 576, + 640, + 704, + 768, + 832, + 896, + 960, + 1024, + 1280, + 1536, + 1792, + 2048, + 2304, + 2560, + 2816, + 3072, + 3328, + 3584, + 3840, + 4096, + 4608, + 5120, + 5632, + 6144, + 6656, + 7168, + 7680, + 8192 + ], + "piecewise_cuda_graph_compiler": "eager", + "torchao_config": "", + "enable_nan_detection": false, + "enable_p2p_check": false, + "triton_attention_reduce_in_fp32": false, + "triton_attention_num_kv_splits": 8, + "triton_attention_split_tile_size": null, + "num_continuous_decode_steps": 1, + "delete_ckpt_after_loading": false, + "enable_memory_saver": false, + "enable_weights_cpu_backup": false, + "enable_draft_weights_cpu_backup": false, + "allow_auto_truncate": false, + "enable_custom_logit_processor": false, + "flashinfer_mla_disable_ragged": false, + "disable_shared_experts_fusion": true, + "disable_chunked_prefix_cache": false, + "disable_fast_image_processor": false, + "keep_mm_feature_on_device": false, + "enable_return_hidden_states": false, + "enable_return_routed_experts": false, + "enable_return_indexer_topk": false, + "scheduler_recv_interval": 1, + "numa_node": null, + "enable_deterministic_inference": false, + "rl_on_policy_target": null, + "enable_attn_tp_input_scattered": false, + "gc_threshold": null, + "enable_nsa_prefill_context_parallel": false, + "nsa_prefill_cp_mode": "in-seq-split", + "enable_fused_qk_norm_rope": false, + "enable_precise_embedding_interpolation": false, + "enable_fused_moe_sum_all_reduce": false, + "enable_prefill_context_parallel": false, + "prefill_cp_mode": "in-seq-split", + "enable_dynamic_batch_tokenizer": false, + "dynamic_batch_tokenizer_batch_size": 32, + "dynamic_batch_tokenizer_batch_timeout": 0.002, + "debug_tensor_dump_output_folder": null, + "debug_tensor_dump_layers": null, + "debug_tensor_dump_input_file": null, + "debug_tensor_dump_inject": false, + "disaggregation_mode": "null", + "disaggregation_transfer_backend": "mooncake", + "disaggregation_bootstrap_port": 8998, + "disaggregation_ib_device": null, + "disaggregation_decode_enable_offload_kvcache": false, + "num_reserved_decode_tokens": 512, + "disaggregation_decode_polling_interval": 1, + "encoder_only": false, + "language_only": false, + "encoder_transfer_backend": "zmq_to_scheduler", + "encoder_urls": [], + "enable_adaptive_dispatch_to_encoder": false, + "enable_over_encoding": false, + "enable_kv_mirror": false, + "custom_weight_loader": [], + "weight_loader_disable_mmap": false, + "remote_instance_weight_loader_seed_instance_ip": null, + "remote_instance_weight_loader_seed_instance_service_port": null, + "remote_instance_weight_loader_send_weights_group_ports": null, + "remote_instance_weight_loader_backend": "nccl", + "remote_instance_weight_loader_start_seed_via_transfer_engine": false, + "engine_info_bootstrap_port": 6789, + "modelexpress_config": null, + "enable_pdmux": false, + "pdmux_config_path": null, + "sm_group_num": 8, + "enable_broadcast_mm_inputs_process": false, + "enable_prefix_mm_cache": false, + "mm_enable_dp_encoder": false, + "mm_process_config": {}, + "limit_mm_data_per_request": null, + "enable_mm_global_cache": false, + "decrypted_config_file": null, + "decrypted_draft_config_file": null, + "forward_hooks": null, + "index_cache_topk_freq": 1, + "index_cache_pattern": null, + "status": "ready", + "max_total_num_tokens": 1593344, + "max_req_input_len": 1048570, + "internal_states": [ + { + "model_path": "/data1/models/DeepSeek-V4-Flash-INT8", + "tokenizer_path": "/data1/models/DeepSeek-V4-Flash-INT8", + "tokenizer_mode": "auto", + "tokenizer_worker_num": 1, + "skip_tokenizer_init": false, + "load_format": "auto", + "model_loader_extra_config": "{}", + "trust_remote_code": true, + "context_length": 1048576, + "is_embedding": false, + "enable_multimodal": null, + "revision": null, + "model_impl": "auto", + "host": "0.0.0.0", + "port": 30000, + "fastapi_root_path": "", + "grpc_mode": false, + "skip_server_warmup": false, + "warmups": null, + "nccl_port": null, + "checkpoint_engine_wait_weights_before_ready": false, + "ssl_keyfile": null, + "ssl_certfile": null, + "ssl_ca_certs": null, + "ssl_keyfile_password": null, + "enable_ssl_refresh": false, + "dtype": "auto", + "quantization": "w8a8_int8", + "quantization_param_path": null, + "kv_cache_dtype": "float16", + "enable_fp32_lm_head": false, + "modelopt_quant": null, + "modelopt_checkpoint_restore_path": null, + "modelopt_checkpoint_save_path": null, + "modelopt_export_path": null, + "quantize_and_serve": false, + "rl_quant_profile": null, + "mem_fraction_static": 0.8, + "max_running_requests": 64, + "max_queued_requests": null, + "max_total_tokens": null, + "chunked_prefill_size": 8192, + "enable_dynamic_chunking": false, + "max_prefill_tokens": 16384, + "prefill_max_requests": null, + "custom_max_context_len": null, + "schedule_policy": "fcfs", + "enable_priority_scheduling": false, + "disable_priority_preemption": false, + "default_priority_value": null, + "abort_on_priority_when_disabled": false, + "schedule_low_priority_values_first": false, + "priority_scheduling_preemption_threshold": 10, + "schedule_conservativeness": 1.0, + "page_size": 256, + "swa_full_tokens_ratio": 0.1, + "disable_hybrid_swa_memory": false, + "radix_eviction_policy": "lru", + "enable_prefill_delayer": false, + "prefill_delayer_max_delay_passes": 30, + "prefill_delayer_token_usage_low_watermark": null, + "prefill_delayer_forward_passes_buckets": null, + "prefill_delayer_wait_seconds_buckets": null, + "device": "cuda", + "tp_size": 8, + "pp_size": 1, + "pp_max_micro_batch_size": 64, + "pp_async_batch_depth": 0, + "stream_interval": 1, + "stream_response_default_include_usage": false, + "incremental_streaming_output": false, + "enable_streaming_session": false, + "random_seed": 228424360, + "constrained_json_whitespace_pattern": null, + "constrained_json_disable_any_whitespace": true, + "watchdog_timeout": 3000000.0, + "soft_watchdog_timeout": null, + "dist_timeout": null, + "download_dir": null, + "model_checksum": null, + "base_gpu_id": 0, + "gpu_id_step": 1, + "sleep_on_idle": false, + "use_ray": false, + "custom_sigquit_handler": null, + "log_level": "info", + "log_level_http": null, + "log_requests": false, + "log_requests_level": 2, + "log_requests_format": "text", + "log_requests_target": null, + "uvicorn_access_log_exclude_prefixes": [], + "crash_dump_folder": null, + "show_time_cost": false, + "enable_metrics": true, + "enable_mfu_metrics": false, + "enable_metrics_for_all_schedulers": false, + "tokenizer_metrics_custom_labels_header": "x-custom-labels", + "tokenizer_metrics_allowed_custom_labels": null, + "extra_metric_labels": null, + "bucket_time_to_first_token": null, + "bucket_inter_token_latency": null, + "bucket_e2e_request_latency": null, + "collect_tokens_histogram": false, + "prompt_tokens_buckets": null, + "generation_tokens_buckets": null, + "gc_warning_threshold_secs": 0.0, + "decode_log_interval": 40, + "enable_request_time_stats_logging": true, + "kv_events_config": null, + "enable_trace": false, + "otlp_traces_endpoint": "localhost:4317", + "export_metrics_to_file": false, + "export_metrics_to_file_dir": null, + "api_key": null, + "admin_api_key": null, + "served_model_name": "/data1/models/DeepSeek-V4-Flash-INT8", + "weight_version": "default", + "chat_template": null, + "hf_chat_template_name": null, + "completion_template": null, + "file_storage_path": "sglang_storage", + "enable_cache_report": false, + "reasoning_parser": "deepseek-v4", + "tool_call_parser": "deepseekv4", + "tool_server": null, + "sampling_defaults": "model", + "dp_size": 1, + "load_balance_method": "round_robin", + "attn_cp_size": 1, + "moe_dp_size": 1, + "dist_init_addr": null, + "nnodes": 1, + "node_rank": 0, + "json_model_override_args": "{}", + "preferred_sampling_params": null, + "enable_lora": null, + "enable_lora_overlap_loading": null, + "max_lora_rank": null, + "lora_target_modules": null, + "lora_paths": null, + "max_loaded_loras": null, + "max_loras_per_batch": 8, + "lora_eviction_policy": "lru", + "lora_backend": "csgmv", + "max_lora_chunk_size": 16, + "experts_shared_outer_loras": null, + "attention_backend": "compressed", + "decode_attention_backend": "compressed", + "prefill_attention_backend": "compressed", + "sampling_backend": "flashinfer", + "grammar_backend": "xgrammar", + "mm_attention_backend": null, + "fp8_gemm_runner_backend": "auto", + "fp4_gemm_runner_backend": "auto", + "nsa_prefill_backend": "klxdsa", + "nsa_decode_backend": "klxdsa", + "disable_flashinfer_autotune": false, + "mamba_backend": "triton", + "speculative_algorithm": "EAGLE", + "speculative_draft_model_path": "/data1/models/DeepSeek-V4-Flash-INT8", + "speculative_draft_model_revision": null, + "speculative_draft_load_format": null, + "speculative_num_steps": 3, + "speculative_eagle_topk": 1, + "speculative_num_draft_tokens": 4, + "speculative_accept_threshold_single": 1.0, + "speculative_accept_threshold_acc": 1.0, + "speculative_token_map": null, + "speculative_attention_mode": "prefill", + "speculative_draft_attention_backend": null, + "speculative_moe_runner_backend": "auto", + "speculative_moe_a2a_backend": null, + "speculative_draft_model_quantization": "w8a8_int8", + "speculative_ngram_min_bfs_breadth": 1, + "speculative_ngram_max_bfs_breadth": 10, + "speculative_ngram_match_type": "BFS", + "speculative_ngram_max_trie_depth": 18, + "speculative_ngram_capacity": 10000000, + "enable_multi_layer_eagle": false, + "ep_size": 8, + "moe_a2a_backend": "none", + "moe_runner_backend": "auto", + "flashinfer_mxfp4_moe_precision": "default", + "enable_hisparse": false, + "enable_flashinfer_allreduce_fusion": false, + "enforce_disable_flashinfer_allreduce_fusion": false, + "enable_aiter_allreduce_fusion": false, + "deepep_mode": "auto", + "ep_num_redundant_experts": 0, + "ep_dispatch_algorithm": null, + "init_expert_location": "trivial", + "enable_eplb": false, + "eplb_algorithm": "auto", + "eplb_rebalance_num_iterations": 1000, + "eplb_rebalance_layers_per_chunk": null, + "eplb_min_rebalancing_utilization_threshold": 1.0, + "expert_distribution_recorder_mode": null, + "expert_distribution_recorder_buffer_size": 1000, + "enable_expert_distribution_metrics": false, + "deepep_config": null, + "moe_dense_tp_size": null, + "elastic_ep_backend": null, + "enable_elastic_expert_backup": false, + "mooncake_ib_device": null, + "max_mamba_cache_size": null, + "mamba_ssm_dtype": null, + "mamba_full_memory_ratio": 0.9, + "mamba_scheduler_strategy": "no_buffer", + "mamba_track_interval": 256, + "linear_attn_backend": "triton", + "linear_attn_decode_backend": null, + "linear_attn_prefill_backend": null, + "enable_hierarchical_cache": false, + "hicache_ratio": 2.0, + "hicache_size": 0, + "hicache_write_policy": "write_through", + "hicache_io_backend": "kernel", + "hicache_mem_layout": "page_first", + "hicache_storage_backend": null, + "hicache_storage_prefetch_policy": "best_effort", + "hicache_storage_backend_extra_config": null, + "hisparse_config": null, + "enable_lmcache": false, + "kt_weight_path": null, + "kt_method": "AMXINT4", + "kt_cpuinfer": null, + "kt_threadpool_count": 2, + "kt_num_gpu_experts": null, + "kt_max_deferred_experts_per_token": null, + "dllm_algorithm": null, + "dllm_algorithm_config": null, + "enable_double_sparsity": false, + "ds_channel_config_path": null, + "ds_heavy_channel_num": 32, + "ds_heavy_token_num": 256, + "ds_heavy_channel_type": "qk", + "ds_sparse_decode_threshold": 4096, + "cpu_offload_gb": 0, + "offload_group_size": -1, + "offload_num_in_group": 1, + "offload_prefetch_step": 1, + "offload_mode": "cpu", + "multi_item_scoring_delimiter": null, + "disable_radix_cache": false, + "cuda_graph_max_bs": 32, + "cuda_graph_bs": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 10, + 12, + 14, + 16, + 18, + 20, + 22, + 24, + 26, + 28, + 30, + 32 + ], + "disable_cuda_graph": false, + "disable_cuda_graph_padding": false, + "enable_profile_cuda_graph": false, + "enable_cudagraph_gc": false, + "enable_layerwise_nvtx_marker": false, + "enable_nccl_nvls": false, + "enable_symm_mem": false, + "disable_flashinfer_cutlass_moe_fp4_allgather": false, + "enable_tokenizer_batch_encode": false, + "disable_tokenizer_batch_decode": false, + "disable_outlines_disk_cache": false, + "disable_custom_all_reduce": true, + "enable_mscclpp": false, + "enable_torch_symm_mem": false, + "pre_warm_nccl": false, + "disable_overlap_schedule": false, + "enable_mixed_chunk": false, + "enable_dp_attention": false, + "enable_dp_lm_head": false, + "enable_two_batch_overlap": false, + "enable_single_batch_overlap": false, + "tbo_token_distribution_threshold": 0.48, + "enable_torch_compile": false, + "disable_piecewise_cuda_graph": true, + "enforce_piecewise_cuda_graph": false, + "enable_torch_compile_debug_mode": false, + "torch_compile_max_bs": 32, + "piecewise_cuda_graph_max_tokens": 8192, + "piecewise_cuda_graph_tokens": [ + 4, + 8, + 12, + 16, + 20, + 24, + 28, + 32, + 48, + 64, + 80, + 96, + 112, + 128, + 144, + 160, + 176, + 192, + 208, + 224, + 240, + 256, + 288, + 320, + 352, + 384, + 416, + 448, + 480, + 512, + 576, + 640, + 704, + 768, + 832, + 896, + 960, + 1024, + 1280, + 1536, + 1792, + 2048, + 2304, + 2560, + 2816, + 3072, + 3328, + 3584, + 3840, + 4096, + 4608, + 5120, + 5632, + 6144, + 6656, + 7168, + 7680, + 8192 + ], + "piecewise_cuda_graph_compiler": "eager", + "torchao_config": "", + "enable_nan_detection": false, + "enable_p2p_check": false, + "triton_attention_reduce_in_fp32": false, + "triton_attention_num_kv_splits": 8, + "triton_attention_split_tile_size": null, + "num_continuous_decode_steps": 1, + "delete_ckpt_after_loading": false, + "enable_memory_saver": false, + "enable_weights_cpu_backup": false, + "enable_draft_weights_cpu_backup": false, + "allow_auto_truncate": false, + "enable_custom_logit_processor": false, + "flashinfer_mla_disable_ragged": false, + "disable_shared_experts_fusion": true, + "disable_chunked_prefix_cache": true, + "disable_fast_image_processor": false, + "keep_mm_feature_on_device": false, + "enable_return_hidden_states": false, + "enable_return_routed_experts": false, + "enable_return_indexer_topk": false, + "scheduler_recv_interval": 1, + "numa_node": null, + "enable_deterministic_inference": false, + "rl_on_policy_target": null, + "enable_attn_tp_input_scattered": false, + "gc_threshold": null, + "enable_nsa_prefill_context_parallel": false, + "nsa_prefill_cp_mode": "in-seq-split", + "enable_fused_qk_norm_rope": false, + "enable_precise_embedding_interpolation": false, + "enable_fused_moe_sum_all_reduce": false, + "enable_prefill_context_parallel": false, + "prefill_cp_mode": "in-seq-split", + "enable_dynamic_batch_tokenizer": false, + "dynamic_batch_tokenizer_batch_size": 32, + "dynamic_batch_tokenizer_batch_timeout": 0.002, + "debug_tensor_dump_output_folder": null, + "debug_tensor_dump_layers": null, + "debug_tensor_dump_input_file": null, + "debug_tensor_dump_inject": false, + "disaggregation_mode": "null", + "disaggregation_transfer_backend": "mooncake", + "disaggregation_bootstrap_port": 8998, + "disaggregation_ib_device": null, + "disaggregation_decode_enable_offload_kvcache": false, + "num_reserved_decode_tokens": 512, + "disaggregation_decode_polling_interval": 1, + "encoder_only": false, + "language_only": false, + "encoder_transfer_backend": "zmq_to_scheduler", + "encoder_urls": [], + "enable_adaptive_dispatch_to_encoder": false, + "enable_over_encoding": false, + "enable_kv_mirror": false, + "custom_weight_loader": [], + "weight_loader_disable_mmap": false, + "remote_instance_weight_loader_seed_instance_ip": null, + "remote_instance_weight_loader_seed_instance_service_port": null, + "remote_instance_weight_loader_send_weights_group_ports": null, + "remote_instance_weight_loader_backend": "nccl", + "remote_instance_weight_loader_start_seed_via_transfer_engine": false, + "engine_info_bootstrap_port": 6789, + "modelexpress_config": null, + "enable_pdmux": false, + "pdmux_config_path": null, + "sm_group_num": 8, + "enable_broadcast_mm_inputs_process": false, + "enable_prefix_mm_cache": false, + "mm_enable_dp_encoder": false, + "mm_process_config": {}, + "limit_mm_data_per_request": null, + "enable_mm_global_cache": false, + "decrypted_config_file": null, + "decrypted_draft_config_file": null, + "forward_hooks": null, + "index_cache_topk_freq": 1, + "index_cache_pattern": null, + "use_mla_backend": false, + "_mx_config_cache": {}, + "_draft_pool_config": { + "full_max_total_num_tokens": 1593344, + "swa_max_total_num_tokens": 159232 + }, + "last_gen_throughput": 304.3209280340091, + "memory_usage": { + "weight": 37.94, + "kvcache": 0, + "token_capacity": 1593344, + "graph": 1.2 + }, + "effective_max_running_requests_per_dp": 64, + "avg_spec_accept_length": 2.8933333333333335 + } + ], + "version": "0.5.10+ee6948cf8" + }, + "duration": 5.34541754698148, + "completed": 10, + "total_input_tokens": 2517, + "total_input_text_tokens": 2517, + "total_input_vision_tokens": 0, + "total_output_tokens": 1404, + "total_output_tokens_retokenized": 1404, + "request_throughput": 1.870761247762007, + "input_throughput": 470.8706060616972, + "output_throughput": 262.6548791857858, + "total_throughput": 733.525485247483, + "mean_e2e_latency_ms": 3372.781996097183, + "median_e2e_latency_ms": 3498.675397480838, + "std_e2e_latency_ms": 1426.9563106485741, + "p90_e2e_latency_ms": 5125.400093087228, + "p99_e2e_latency_ms": 5314.590020017349, + "mean_ttft_ms": 478.8548628857825, + "median_ttft_ms": 506.4399079710711, + "std_ttft_ms": 82.89120389713447, + "p99_ttft_ms": 507.2712521097855, + "mean_tpot_ms": 22.334078483797782, + "median_tpot_ms": 21.948128755653826, + "std_tpot_ms": 3.1100348959353536, + "p99_tpot_ms": 29.154062995221466, + "mean_itl_ms": 20.75981864563044, + "median_itl_ms": 18.560815330905218, + "std_itl_ms": 11.544034313142475, + "p95_itl_ms": 30.416983005125076, + "p99_itl_ms": 76.09861597185954, + "concurrency": 6.309669855447998, + "accept_length": 2.8933333333333335, + "max_output_tokens_per_s": 396.0, + "max_concurrent_requests": 10, + "input_lens": [ + 38, + 236, + 397, + 73, + 256, + 394, + 204, + 134, + 336, + 449 + ], + "output_lens": [ + 145, + 130, + 205, + 72, + 238, + 253, + 135, + 26, + 179, + 21 + ], + "ttfts": [ + 0.23018694896018133, + 0.5072830729768611, + 0.5071517300093547, + 0.506980306992773, + 0.5067555939895101, + 0.5065123789827339, + 0.5063674369594082, + 0.5061119620222598, + 0.5057157859555446, + 0.5054834120091982 + ], + "itls": [ + [ + 0.1265941219899105, + 0.1265941219899105, + 0.1265941219899105, + 0.1265941219899105, + 0.03003738750703633, + 0.03003738750703633, + 0.020075495665272076, + 0.020075495665272076, + 0.020075495665272076, + 0.020034517995857943, + 0.020034517995857943, + 0.020034517995857943, + 0.020025947674488027, + 0.020025947674488027, + 0.020025947674488027, + 0.01982437400147319, + 0.01982437400147319, + 0.01982437400147319, + 0.030580375489080325, + 0.030580375489080325, + 0.015474200758035295, + 0.015474200758035295, + 0.015474200758035295, + 0.015474200758035295, + 0.02975128698744811, + 0.02975128698744811, + 0.018891388666816056, + 0.018891388666816056, + 0.018891388666816056, + 0.030985667331454653, + 0.030985667331454653, + 0.030985667331454653, + 0.01450748200295493, + 0.01450748200295493, + 0.01450748200295493, + 0.01450748200295493, + 0.02843167950049974, + 0.02843167950049974, + 0.019097025661418836, + 0.019097025661418836, + 0.019097025661418836, + 0.019033391009240102, + 0.019033391009240102, + 0.019033391009240102, + 0.02840644898242317, + 0.02840644898242317, + 0.0287785355176311, + 0.0287785355176311, + 0.01427662099013105, + 0.01427662099013105, + 0.01427662099013105, + 0.01427662099013105, + 0.028972148516913876, + 0.028972148516913876, + 0.0198178769981799, + 0.0198178769981799, + 0.0198178769981799, + 0.019058960994395118, + 0.019058960994395118, + 0.019058960994395118, + 0.014342486509121954, + 0.014342486509121954, + 0.014342486509121954, + 0.014342486509121954, + 0.028646462480537593, + 0.028646462480537593, + 0.01456742275331635, + 0.01456742275331635, + 0.01456742275331635, + 0.01456742275331635, + 0.01943501700103904, + 0.01943501700103904, + 0.01943501700103904, + 0.02791283550322987, + 0.02791283550322987, + 0.013792212252155878, + 0.013792212252155878, + 0.013792212252155878, + 0.013792212252155878, + 0.018324121988068025, + 0.018324121988068025, + 0.018324121988068025, + 0.018392249670190115, + 0.018392249670190115, + 0.018392249670190115, + 0.018568886327557266, + 0.018568886327557266, + 0.018568886327557266, + 0.018351545673795044, + 0.018351545673795044, + 0.018351545673795044, + 0.02752352401148528, + 0.02752352401148528, + 0.013863828251487575, + 0.013863828251487575, + 0.013863828251487575, + 0.013863828251487575, + 0.018301511649042368, + 0.018301511649042368, + 0.018301511649042368, + 0.027680501516442746, + 0.027680501516442746, + 0.01841827033786103, + 0.01841827033786103, + 0.01841827033786103, + 0.01374593649234157, + 0.01374593649234157, + 0.01374593649234157, + 0.01374593649234157, + 0.018055193999316543, + 0.018055193999316543, + 0.018055193999316543, + 0.014001160496263765, + 0.014001160496263765, + 0.014001160496263765, + 0.014001160496263765, + 0.018516544679490227, + 0.018516544679490227, + 0.018516544679490227, + 0.018641953337161492, + 0.018641953337161492, + 0.018641953337161492, + 0.01858119231959184, + 0.01858119231959184, + 0.01858119231959184, + 0.018560895676879834, + 0.018560895676879834, + 0.018560895676879834, + 0.027825927507365122, + 0.027825927507365122, + 0.027690790477208793, + 0.027690790477208793, + 0.014033740997547284, + 0.014033740997547284, + 0.014033740997547284, + 0.014033740997547284, + 0.028556485020089895, + 0.028556485020089895, + 0.018292231000183772, + 0.018292231000183772, + 0.018292231000183772, + 0.03276129499621069, + 0.03276129499621069, + 0.03276129499621069 + ], + [ + 0.114176959003089, + 0.114176959003089, + 0.02001935833444198, + 0.02001935833444198, + 0.02001935833444198, + 0.020075112329019856, + 0.020075112329019856, + 0.020075112329019856, + 0.02003466134192422, + 0.02003466134192422, + 0.02003466134192422, + 0.020026450996131945, + 0.020026450996131945, + 0.020026450996131945, + 0.02973513101460412, + 0.02973513101460412, + 0.030412352993153036, + 0.030412352993153036, + 0.015535941754933447, + 0.015535941754933447, + 0.015535941754933447, + 0.015535941754933447, + 0.02982658799737692, + 0.02982658799737692, + 0.018890631986626733, + 0.018890631986626733, + 0.018890631986626733, + 0.03098562400555238, + 0.03098562400555238, + 0.03098562400555238, + 0.019343592672764014, + 0.019343592672764014, + 0.019343592672764014, + 0.018950106323851895, + 0.018950106323851895, + 0.018950106323851895, + 0.019099269004072994, + 0.019099269004072994, + 0.019099269004072994, + 0.019034251008027542, + 0.019034251008027542, + 0.019034251008027542, + 0.01420391199644655, + 0.01420391199644655, + 0.01420391199644655, + 0.01420391199644655, + 0.01439423524425365, + 0.01439423524425365, + 0.01439423524425365, + 0.01439423524425365, + 0.028566046996274963, + 0.028566046996274963, + 0.01930476234216864, + 0.01930476234216864, + 0.01930476234216864, + 0.019809113660206396, + 0.019809113660206396, + 0.019809113660206396, + 0.014296345761977136, + 0.014296345761977136, + 0.014296345761977136, + 0.014296345761977136, + 0.02867897297255695, + 0.02867897297255695, + 0.019099251677592594, + 0.019099251677592594, + 0.019099251677592594, + 0.019382666000941146, + 0.019382666000941146, + 0.019382666000941146, + 0.02924422750947997, + 0.02924422750947997, + 0.027913359983358532, + 0.027913359983358532, + 0.013784522248897702, + 0.013784522248897702, + 0.013784522248897702, + 0.013784522248897702, + 0.02746082251542248, + 0.02746082251542248, + 0.01840783333561073, + 0.01840783333561073, + 0.01840783333561073, + 0.01856821965581427, + 0.01856821965581427, + 0.01856821965581427, + 0.01835258233283336, + 0.01835258233283336, + 0.01835258233283336, + 0.018347706005442888, + 0.018347706005442888, + 0.018347706005442888, + 0.018484461004845798, + 0.018484461004845798, + 0.018484461004845798, + 0.018302411655895412, + 0.018302411655895412, + 0.018302411655895412, + 0.027679786493536085, + 0.027679786493536085, + 0.027627180505078286, + 0.027627180505078286, + 0.01375247175747063, + 0.01375247175747063, + 0.01375247175747063, + 0.01375247175747063, + 0.0270681754918769, + 0.0270681754918769, + 0.01866779732517898, + 0.01866779732517898, + 0.01866779732517898, + 0.018516544679490227, + 0.018516544679490227, + 0.018516544679490227, + 0.013980599993374199, + 0.013980599993374199, + 0.013980599993374199, + 0.013980599993374199, + 0.027871913509443402, + 0.027871913509443402, + 0.018560129004375387, + 0.018560129004375387, + 0.018560129004375387, + 0.018560815330905218, + 0.018560815330905218, + 0.018560815330905218, + 0.027690300485119224, + 0.027690300485119224, + 0.05595995101612061 + ], + [ + 0.07610034267418087, + 0.07610034267418087, + 0.07610034267418087, + 0.02001753800626223, + 0.02001753800626223, + 0.02001753800626223, + 0.020074322334645938, + 0.020074322334645938, + 0.020074322334645938, + 0.015025673492345959, + 0.015025673492345959, + 0.015025673492345959, + 0.015025673492345959, + 0.03003843649639748, + 0.03003843649639748, + 0.01982440067998444, + 0.01982440067998444, + 0.01982440067998444, + 0.02034523299274345, + 0.02034523299274345, + 0.02034523299274345, + 0.030977007001638412, + 0.030977007001638412, + 0.029805967991705984, + 0.029805967991705984, + 0.014167338755214587, + 0.014167338755214587, + 0.014167338755214587, + 0.014167338755214587, + 0.046478810982080176, + 0.046478810982080176, + 0.01934337633429095, + 0.01934337633429095, + 0.01934337633429095, + 0.0189525830016161, + 0.0189525830016161, + 0.0189525830016161, + 0.014323239010991529, + 0.014323239010991529, + 0.014323239010991529, + 0.014323239010991529, + 0.028550161485327408, + 0.028550161485327408, + 0.018938299675937742, + 0.018938299675937742, + 0.018938299675937742, + 0.019176949999139953, + 0.019176949999139953, + 0.019176949999139953, + 0.028566726978169754, + 0.028566726978169754, + 0.028956993512110785, + 0.028956993512110785, + 0.014857275251415558, + 0.014857275251415558, + 0.014857275251415558, + 0.014857275251415558, + 0.028592451504664496, + 0.028592451504664496, + 0.01912001532036811, + 0.01912001532036811, + 0.01912001532036811, + 0.019098964675019186, + 0.019098964675019186, + 0.019098964675019186, + 0.014537554743583314, + 0.014537554743583314, + 0.014537554743583314, + 0.014537554743583314, + 0.029245732497656718, + 0.029245732497656718, + 0.018609130017769832, + 0.018609130017769832, + 0.018609130017769832, + 0.018378569666917127, + 0.018378569666917127, + 0.018378569666917127, + 0.027461227495223284, + 0.027461227495223284, + 0.013806014991132542, + 0.013806014991132542, + 0.013806014991132542, + 0.013806014991132542, + 0.013926397252362221, + 0.013926397252362221, + 0.013926397252362221, + 0.013926397252362221, + 0.01835247567699601, + 0.01835247567699601, + 0.01835247567699601, + 0.01834764933058371, + 0.01834764933058371, + 0.01834764933058371, + 0.013863673244486563, + 0.013863673244486563, + 0.013863673244486563, + 0.013863673244486563, + 0.027453012502519414, + 0.027453012502519414, + 0.018453627669562895, + 0.018453627669562895, + 0.018453627669562895, + 0.018418410327285528, + 0.018418410327285528, + 0.018418410327285528, + 0.027505588514031842, + 0.027505588514031842, + 0.013533967750845477, + 0.013533967750845477, + 0.013533967750845477, + 0.013533967750845477, + 0.014000577997649089, + 0.014000577997649089, + 0.014000577997649089, + 0.014000577997649089, + 0.018516601334946852, + 0.018516601334946852, + 0.018516601334946852, + 0.018641606671735644, + 0.018641606671735644, + 0.018641606671735644, + 0.013935844239313155, + 0.013935844239313155, + 0.013935844239313155, + 0.013935844239313155, + 0.02784075849922374, + 0.02784075849922374, + 0.018561025344145794, + 0.018561025344145794, + 0.018561025344145794, + 0.01846017032706489, + 0.01846017032706489, + 0.01846017032706489, + 0.02801839599851519, + 0.02801839599851519, + 0.014315253254608251, + 0.014315253254608251, + 0.014315253254608251, + 0.014315253254608251, + 0.013719310751184821, + 0.013719310751184821, + 0.013719310751184821, + 0.013719310751184821, + 0.032736671001960836, + 0.032736671001960836, + 0.032736671001960836, + 0.01810533132326479, + 0.01810533132326479, + 0.01810533132326479, + 0.025817634508712217, + 0.025817634508712217, + 0.017274240672122687, + 0.017274240672122687, + 0.017274240672122687, + 0.012808780491468497, + 0.012808780491468497, + 0.012808780491468497, + 0.012808780491468497, + 0.017103748007987935, + 0.017103748007987935, + 0.017103748007987935, + 0.0260774539783597, + 0.0260774539783597, + 0.012770477012963966, + 0.012770477012963966, + 0.012770477012963966, + 0.012770477012963966, + 0.012633724749321118, + 0.012633724749321118, + 0.012633724749321118, + 0.012633724749321118, + 0.016969308664556593, + 0.016969308664556593, + 0.016969308664556593, + 0.017546184652019292, + 0.017546184652019292, + 0.017546184652019292, + 0.02627176101668738, + 0.02627176101668738, + 0.01677496867099156, + 0.01677496867099156, + 0.01677496867099156, + 0.012476857242290862, + 0.012476857242290862, + 0.012476857242290862, + 0.012476857242290862, + 0.016706993997407455, + 0.016706993997407455, + 0.016706993997407455, + 0.02511487199808471, + 0.02511487199808471, + 0.012620112000149675, + 0.012620112000149675, + 0.012620112000149675, + 0.012620112000149675, + 0.013204801754909568, + 0.013204801754909568, + 0.013204801754909568, + 0.013204801754909568, + 0.016931711327439796, + 0.016931711327439796, + 0.016931711327439796, + 0.025843739014817402, + 0.025843739014817402 + ], + [ + 0.07609820265012483, + 0.07609820265012483, + 0.07609820265012483, + 0.020024441337833803, + 0.020024441337833803, + 0.020024441337833803, + 0.015056459509651177, + 0.015056459509651177, + 0.015056459509651177, + 0.015056459509651177, + 0.03005173648125492, + 0.03005173648125492, + 0.015019418249721639, + 0.015019418249721639, + 0.015019418249721639, + 0.015019418249721639, + 0.019824344334968675, + 0.019824344334968675, + 0.019824344334968675, + 0.020387480326462537, + 0.020387480326462537, + 0.020387480326462537, + 0.020631657680496573, + 0.020631657680496573, + 0.020631657680496573, + 0.01487621849810239, + 0.01487621849810239, + 0.01487621849810239, + 0.01487621849810239, + 0.028335707494989038, + 0.028335707494989038, + 0.023239640504471026, + 0.023239640504471026, + 0.023239640504471026, + 0.023239640504471026, + 0.0193434563310196, + 0.0193434563310196, + 0.0193434563310196, + 0.028431184502551332, + 0.028431184502551332, + 0.014322734248708002, + 0.014322734248708002, + 0.014322734248708002, + 0.014322734248708002, + 0.01903337399320056, + 0.01903337399320056, + 0.01903337399320056, + 0.01893743300267185, + 0.01893743300267185, + 0.01893743300267185, + 0.01917593334413444, + 0.01917593334413444, + 0.01917593334413444, + 0.028568201989401132, + 0.028568201989401132, + 0.01930442900629714, + 0.01930442900629714, + 0.01930442900629714, + 0.014869755497784354, + 0.014869755497784354, + 0.014869755497784354, + 0.014869755497784354, + 0.028589651483343914, + 0.028589651483343914, + 0.019121012010145932, + 0.019121012010145932, + 0.019121012010145932, + 0.01909933132507528, + 0.01909933132507528, + 0.01909933132507528, + 0.0582002800074406 + ], + [ + 0.07609824265819043, + 0.07609824265819043, + 0.07609824265819043, + 0.015011793249868788, + 0.015011793249868788, + 0.015011793249868788, + 0.015011793249868788, + 0.020074732679252822, + 0.020074732679252822, + 0.020074732679252822, + 0.015025845990749076, + 0.015025845990749076, + 0.015025845990749076, + 0.015025845990749076, + 0.030038921511732042, + 0.030038921511732042, + 0.019824267326233286, + 0.019824267326233286, + 0.019824267326233286, + 0.020395867332505684, + 0.020395867332505684, + 0.020395867332505684, + 0.03094582151970826, + 0.03094582151970826, + 0.014877470748615451, + 0.014877470748615451, + 0.014877470748615451, + 0.014877470748615451, + 0.014167653993354179, + 0.014167653993354179, + 0.014167653993354179, + 0.014167653993354179, + 0.030986197331609826, + 0.030986197331609826, + 0.030986197331609826, + 0.01934319967404008, + 0.01934319967404008, + 0.01934319967404008, + 0.014215631992556155, + 0.014215631992556155, + 0.014215631992556155, + 0.014215631992556155, + 0.02864447850151919, + 0.02864447850151919, + 0.01903398767656957, + 0.01903398767656957, + 0.01903398767656957, + 0.018937196330322575, + 0.018937196330322575, + 0.018937196330322575, + 0.028763925016392022, + 0.028763925016392022, + 0.02856709697516635, + 0.02856709697516635, + 0.014478914003120735, + 0.014478914003120735, + 0.014478914003120735, + 0.014478914003120735, + 0.029714685515500605, + 0.029714685515500605, + 0.019061427660441648, + 0.019061427660441648, + 0.019061427660441648, + 0.019120605332621683, + 0.019120605332621683, + 0.019120605332621683, + 0.014324548756121658, + 0.014324548756121658, + 0.014324548756121658, + 0.014324548756121658, + 0.0290751994762104, + 0.0290751994762104, + 0.019497701665386558, + 0.019497701665386558, + 0.019497701665386558, + 0.01860897334214921, + 0.01860897334214921, + 0.01860897334214921, + 0.02756879449589178, + 0.02756879449589178, + 0.02748415298992768, + 0.02748415298992768, + 0.01379515475127846, + 0.01379515475127846, + 0.01379515475127846, + 0.01379515475127846, + 0.02785251452587545, + 0.02785251452587545, + 0.018352062324993312, + 0.018352062324993312, + 0.018352062324993312, + 0.01834822599388038, + 0.01834822599388038, + 0.01834822599388038, + 0.01386378325696569, + 0.01386378325696569, + 0.01386378325696569, + 0.01386378325696569, + 0.02745306250290014, + 0.02745306250290014, + 0.018453344334072124, + 0.018453344334072124, + 0.018453344334072124, + 0.018418603654329974, + 0.018418603654329974, + 0.018418603654329974, + 0.02750637350254692, + 0.02750637350254692, + 0.027067835500929505, + 0.027067835500929505, + 0.014000943003338762, + 0.014000943003338762, + 0.014000943003338762, + 0.014000943003338762, + 0.027774896996561438, + 0.027774896996561438, + 0.01864148332970217, + 0.01864148332970217, + 0.01864148332970217, + 0.018581362344169367, + 0.018581362344169367, + 0.018581362344169367, + 0.013920626748586074, + 0.013920626748586074, + 0.013920626748586074, + 0.013920626748586074, + 0.027841037983307615, + 0.027841037983307615, + 0.018460453681958217, + 0.018460453681958217, + 0.018460453681958217, + 0.018680113988618057, + 0.018680113988618057, + 0.018680113988618057, + 0.028647751518292353, + 0.028647751518292353, + 0.02743947648559697, + 0.02743947648559697, + 0.024545218504499644, + 0.024545218504499644, + 0.024545218504499644, + 0.024545218504499644, + 0.027157786476891488, + 0.027157786476891488, + 0.017211266347051907, + 0.017211266347051907, + 0.017211266347051907, + 0.017274483999547858, + 0.017274483999547858, + 0.017274483999547858, + 0.012808570492779836, + 0.012808570492779836, + 0.012808570492779836, + 0.012808570492779836, + 0.02565537701593712, + 0.02565537701593712, + 0.017431950002598267, + 0.017431950002598267, + 0.017431950002598267, + 0.016987925317759316, + 0.016987925317759316, + 0.016987925317759316, + 0.025259989517508075, + 0.025259989517508075, + 0.012736934004351497, + 0.012736934004351497, + 0.012736934004351497, + 0.012736934004351497, + 0.013151180988643318, + 0.013151180988643318, + 0.013151180988643318, + 0.013151180988643318, + 0.01752863434376195, + 0.01752863434376195, + 0.01752863434376195, + 0.01677370866915832, + 0.01677370866915832, + 0.01677370866915832, + 0.01247714224155061, + 0.01247714224155061, + 0.01247714224155061, + 0.01247714224155061, + 0.025057951017515734, + 0.025057951017515734, + 0.016742911325612415, + 0.016742911325612415, + 0.016742911325612415, + 0.016827172657940537, + 0.016827172657940537, + 0.016827172657940537, + 0.0176063990026402, + 0.0176063990026402, + 0.0176063990026402, + 0.01693138466604675, + 0.01693138466604675, + 0.01693138466604675, + 0.012955810001585633, + 0.012955810001585633, + 0.012955810001585633, + 0.012955810001585633, + 0.016913124340741586, + 0.016913124340741586, + 0.016913124340741586, + 0.015995322338615853, + 0.015995322338615853, + 0.015995322338615853, + 0.012007721743430011, + 0.012007721743430011, + 0.012007721743430011, + 0.012007721743430011, + 0.024051369488006458, + 0.024051369488006458, + 0.015974845349167783, + 0.015974845349167783, + 0.015974845349167783, + 0.015958434999144327, + 0.015958434999144327, + 0.015958434999144327, + 0.015792935659798484, + 0.015792935659798484, + 0.015792935659798484, + 0.01592183733979861, + 0.01592183733979861, + 0.01592183733979861, + 0.011986501252977178, + 0.011986501252977178, + 0.011986501252977178, + 0.011986501252977178, + 0.01585641598406558, + 0.01585641598406558, + 0.01585641598406558, + 0.015924710663966835, + 0.015924710663966835, + 0.015924710663966835, + 0.01634039101190865, + 0.01634039101190865, + 0.01634039101190865 + ], + [ + 0.057074277006904595, + 0.057074277006904595, + 0.057074277006904595, + 0.057074277006904595, + 0.030024631996639073, + 0.030024631996639073, + 0.020074642340963084, + 0.020074642340963084, + 0.020074642340963084, + 0.020034064655192196, + 0.020034064655192196, + 0.020034064655192196, + 0.015019408252555877, + 0.015019408252555877, + 0.015019408252555877, + 0.015019408252555877, + 0.02973640151321888, + 0.02973640151321888, + 0.020345932998073597, + 0.020345932998073597, + 0.020345932998073597, + 0.020652821326317888, + 0.020652821326317888, + 0.020652821326317888, + 0.014902361246640794, + 0.014902361246640794, + 0.014902361246640794, + 0.014902361246640794, + 0.014167511501000263, + 0.014167511501000263, + 0.014167511501000263, + 0.014167511501000263, + 0.04647939599817619, + 0.04647939599817619, + 0.019343009334988892, + 0.019343009334988892, + 0.019343009334988892, + 0.01895378966582939, + 0.01895378966582939, + 0.01895378966582939, + 0.014322384260594845, + 0.014322384260594845, + 0.014322384260594845, + 0.014322384260594845, + 0.02855006148456596, + 0.02855006148456596, + 0.018938185996375978, + 0.018938185996375978, + 0.018938185996375978, + 0.019176096674830962, + 0.019176096674830962, + 0.019176096674830962, + 0.019044518005102873, + 0.019044518005102873, + 0.019044518005102873, + 0.019305008987430483, + 0.019305008987430483, + 0.019305008987430483, + 0.014857510250294581, + 0.014857510250294581, + 0.014857510250294581, + 0.014857510250294581, + 0.0190613410086371, + 0.0190613410086371, + 0.0190613410086371, + 0.019120295338022213, + 0.019120295338022213, + 0.019120295338022213, + 0.0143242962367367, + 0.0143242962367367, + 0.0143242962367367, + 0.0143242962367367, + 0.029075569502310827, + 0.029075569502310827, + 0.019497531349770725, + 0.019497531349770725, + 0.019497531349770725, + 0.018608823651447892, + 0.018608823651447892, + 0.018608823651447892, + 0.018379226346344996, + 0.018379226346344996, + 0.018379226346344996, + 0.018320148325680446, + 0.018320148325680446, + 0.018320148325680446, + 0.01379665250715334, + 0.01379665250715334, + 0.01379665250715334, + 0.01379665250715334, + 0.01856846265339603, + 0.01856846265339603, + 0.01856846265339603, + 0.018352199345827103, + 0.018352199345827103, + 0.018352199345827103, + 0.018347878998611122, + 0.018347878998611122, + 0.018347878998611122, + 0.02772726200055331, + 0.02772726200055331, + 0.013726471239351667, + 0.013726471239351667, + 0.013726471239351667, + 0.013726471239351667, + 0.018453674002860982, + 0.018453674002860982, + 0.018453674002860982, + 0.027627600502455607, + 0.027627600502455607, + 0.013753056759014726, + 0.013753056759014726, + 0.013753056759014726, + 0.013753056759014726, + 0.013533932986319996, + 0.013533932986319996, + 0.013533932986319996, + 0.013533932986319996, + 0.018667823673846822, + 0.018667823673846822, + 0.018667823673846822, + 0.01851649465970695, + 0.01851649465970695, + 0.01851649465970695, + 0.027962180523900315, + 0.027962180523900315, + 0.01858143232918034, + 0.01858143232918034, + 0.01858143232918034, + 0.01392033399315551, + 0.01392033399315551, + 0.01392033399315551, + 0.01392033399315551, + 0.01856105534049372, + 0.01856105534049372, + 0.01856105534049372, + 0.027690295508364215, + 0.027690295508364215, + 0.014009808248374611, + 0.014009808248374611, + 0.014009808248374611, + 0.014009808248374611, + 0.014324050745926797, + 0.014324050745926797, + 0.014324050745926797, + 0.014324050745926797, + 0.018292734341230243, + 0.018292734341230243, + 0.018292734341230243, + 0.03272628099269544, + 0.03272628099269544, + 0.03272628099269544, + 0.027157796488609165, + 0.027157796488609165, + 0.017211413010954857, + 0.017211413010954857, + 0.017211413010954857, + 0.012955822996445931, + 0.012955822996445931, + 0.012955822996445931, + 0.012955822996445931, + 0.017078191003141303, + 0.017078191003141303, + 0.017078191003141303, + 0.02565545149263926, + 0.02565545149263926, + 0.026133759994991124, + 0.026133759994991124, + 0.012745351748890243, + 0.012745351748890243, + 0.012745351748890243, + 0.012745351748890243, + 0.02526378951733932, + 0.02526378951733932, + 0.016981848670790594, + 0.016981848670790594, + 0.016981848670790594, + 0.01753506433063497, + 0.01753506433063497, + 0.01753506433063497, + 0.013146101002348587, + 0.013146101002348587, + 0.013146101002348587, + 0.013146101002348587, + 0.025147862499579787, + 0.025147862499579787, + 0.016644779650960118, + 0.016644779650960118, + 0.016644779650960118, + 0.01670563401421532, + 0.01670563401421532, + 0.01670563401421532, + 0.025114176998613402, + 0.025114176998613402, + 0.025240779505111277, + 0.025240779505111277, + 0.01760628231568262, + 0.01760628231568262, + 0.01760628231568262, + 0.016931691012966137, + 0.016931691012966137, + 0.016931691012966137, + 0.01727279700571671, + 0.01727279700571671, + 0.01727279700571671, + 0.016912257318229724, + 0.016912257318229724, + 0.016912257318229724, + 0.01598840901472916, + 0.01598840901472916, + 0.01598840901472916, + 0.016010235665210832, + 0.016010235665210832, + 0.016010235665210832, + 0.016034339651620638, + 0.016034339651620638, + 0.016034339651620638, + 0.015974988675831508, + 0.015974988675831508, + 0.015974988675831508, + 0.023938142490806058, + 0.023938142490806058, + 0.023689378518611193, + 0.023689378518611193, + 0.01592199399601668, + 0.01592199399601668, + 0.01592199399601668, + 0.01598192499174426, + 0.01598192499174426, + 0.01598192499174426, + 0.01586189299511413, + 0.01586189299511413, + 0.01586189299511413, + 0.015926023673576612, + 0.015926023673576612, + 0.015926023673576612, + 0.016315380674010765, + 0.016315380674010765, + 0.016315380674010765, + 0.017069103002237778, + 0.017069103002237778, + 0.017069103002237778, + 0.015235118994799754, + 0.015235118994799754, + 0.015235118994799754, + 0.015159054659307003, + 0.015159054659307003, + 0.015159054659307003, + 0.022672360995784402, + 0.022672360995784402, + 0.023076677520293742, + 0.023076677520293742 + ], + [ + 0.07599885431894411, + 0.07599885431894411, + 0.07599885431894411, + 0.020025841678337503, + 0.020025841678337503, + 0.020025841678337503, + 0.020075395664510626, + 0.020075395664510626, + 0.020075395664510626, + 0.020034617996619392, + 0.020034617996619392, + 0.020034617996619392, + 0.020026374336642522, + 0.020026374336642522, + 0.020026374336642522, + 0.019823340660271544, + 0.019823340660271544, + 0.019823340660271544, + 0.030416983005125076, + 0.030416983005125076, + 0.03106233850121498, + 0.03106233850121498, + 0.019885292005104322, + 0.019885292005104322, + 0.019885292005104322, + 0.018892501985343795, + 0.018892501985343795, + 0.018892501985343795, + 0.030985270665648084, + 0.030985270665648084, + 0.030985270665648084, + 0.019343586017688114, + 0.019343586017688114, + 0.019343586017688114, + 0.018950182983341318, + 0.018950182983341318, + 0.018950182983341318, + 0.019099209011377145, + 0.019099209011377145, + 0.019099209011377145, + 0.01903421765503784, + 0.01903421765503784, + 0.01903421765503784, + 0.018937779338254284, + 0.018937779338254284, + 0.018937779338254284, + 0.02875587999005802, + 0.02875587999005802, + 0.02856260701082647, + 0.02856260701082647, + 0.019304305334420253, + 0.019304305334420253, + 0.019304305334420253, + 0.019817157338062923, + 0.019817157338062923, + 0.019817157338062923, + 0.019061451001713674, + 0.019061451001713674, + 0.019061451001713674, + 0.019113485332733642, + 0.019113485332733642, + 0.019113485332733642, + 0.019104608334600925, + 0.019104608334600925, + 0.019104608334600925, + 0.02907446399331093, + 0.02907446399331093, + 0.014618368746596389, + 0.014618368746596389, + 0.014618368746596389, + 0.014618368746596389, + 0.018610139998296898, + 0.018610139998296898, + 0.018610139998296898, + 0.02757459500571713, + 0.02757459500571713, + 0.027460306999273598, + 0.027460306999273598, + 0.01380542025435716, + 0.01380542025435716, + 0.01380542025435716, + 0.01380542025435716, + 0.027853768988279626, + 0.027853768988279626, + 0.0183525126776658, + 0.0183525126776658, + 0.0183525126776658, + 0.018347032329378028, + 0.018347032329378028, + 0.018347032329378028, + 0.018484470997160923, + 0.018484470997160923, + 0.018484470997160923, + 0.01830341199335332, + 0.01830341199335332, + 0.01830341199335332, + 0.01845283400810634, + 0.01845283400810634, + 0.01845283400810634, + 0.018417870334815234, + 0.018417870334815234, + 0.018417870334815234, + 0.027498273499077186, + 0.027498273499077186, + 0.027074395504314452, + 0.027074395504314452, + 0.018667700661656756, + 0.018667700661656756, + 0.018667700661656756, + 0.01851662133897965, + 0.01851662133897965, + 0.01851662133897965, + 0.018641199994211394, + 0.018641199994211394, + 0.018641199994211394, + 0.018580482341349125, + 0.018580482341349125, + 0.018580482341349125, + 0.018559868990754087, + 0.018559868990754087, + 0.018559868990754087, + 0.018557828676421195, + 0.018557828676421195, + 0.018557828676421195, + 0.018460286994619917, + 0.018460286994619917, + 0.018460286994619917, + 0.018637193328080077, + 0.018637193328080077, + 0.018637193328080077, + 0.028849135007476434, + 0.028849135007476434, + 0.027445936488220468, + 0.027445936488220468 + ], + [ + 0.07609848600501816, + 0.07609848600501816, + 0.07609848600501816, + 0.02001590799773112, + 0.02001590799773112, + 0.02001590799773112, + 0.020074725985371817, + 0.020074725985371817, + 0.020074725985371817, + 0.015025570755824447, + 0.015025570755824447, + 0.015025570755824447, + 0.015025570755824447, + 0.0200260576675646, + 0.0200260576675646, + 0.0200260576675646, + 0.014868118247250095, + 0.014868118247250095, + 0.014868118247250095, + 0.014868118247250095, + 0.01529539775219746, + 0.01529539775219746, + 0.01529539775219746, + 0.01529539775219746, + 0.0618297120090574 + ], + [ + 0.11415219900663942, + 0.11415219900663942, + 0.020037391669272136, + 0.020037391669272136, + 0.020037391669272136, + 0.015055509502417408, + 0.015055509502417408, + 0.015055509502417408, + 0.015055509502417408, + 0.020034414328013856, + 0.020034414328013856, + 0.020034414328013856, + 0.020025807665660977, + 0.020025807665660977, + 0.020025807665660977, + 0.01982415100792423, + 0.01982415100792423, + 0.01982415100792423, + 0.030516334489220753, + 0.030516334489220753, + 0.015466665499843657, + 0.015466665499843657, + 0.015466665499843657, + 0.015466665499843657, + 0.019884118674478184, + 0.019884118674478184, + 0.019884118674478184, + 0.028335032984614372, + 0.028335032984614372, + 0.023239655507495627, + 0.023239655507495627, + 0.023239655507495627, + 0.023239655507495627, + 0.014507339743431658, + 0.014507339743431658, + 0.014507339743431658, + 0.014507339743431658, + 0.01895076932851225, + 0.01895076932851225, + 0.01895076932851225, + 0.019099789011913042, + 0.019099789011913042, + 0.019099789011913042, + 0.028550306480610743, + 0.028550306480610743, + 0.018938216341969866, + 0.018938216341969866, + 0.018938216341969866, + 0.019193043660682935, + 0.019193043660682935, + 0.019193043660682935, + 0.019044294662307948, + 0.019044294662307948, + 0.019044294662307948, + 0.019304565348041553, + 0.019304565348041553, + 0.019304565348041553, + 0.014856965237413533, + 0.014856965237413533, + 0.014856965237413533, + 0.014856965237413533, + 0.014296558263595216, + 0.014296558263595216, + 0.014296558263595216, + 0.014296558263595216, + 0.01911949533193062, + 0.01911949533193062, + 0.01911949533193062, + 0.019099241665874917, + 0.019099241665874917, + 0.019099241665874917, + 0.029074459482217208, + 0.029074459482217208, + 0.01949675833263124, + 0.01949675833263124, + 0.01949675833263124, + 0.013956615002825856, + 0.013956615002825856, + 0.013956615002825856, + 0.013956615002825856, + 0.018378923006821424, + 0.018378923006821424, + 0.018378923006821424, + 0.027460777491796762, + 0.027460777491796762, + 0.013805887501803227, + 0.013805887501803227, + 0.013805887501803227, + 0.013805887501803227, + 0.013926447005360387, + 0.013926447005360387, + 0.013926447005360387, + 0.013926447005360387, + 0.01835258931775267, + 0.01835258931775267, + 0.01835258931775267, + 0.018347795664643247, + 0.018347795664643247, + 0.018347795664643247, + 0.013863318512449041, + 0.013863318512449041, + 0.013863318512449041, + 0.013863318512449041, + 0.027453502494608983, + 0.027453502494608983, + 0.01845339099721362, + 0.01845339099721362, + 0.01845339099721362, + 0.018418139991505694, + 0.018418139991505694, + 0.018418139991505694, + 0.027505743521032855, + 0.027505743521032855, + 0.027068145980592817, + 0.027068145980592817, + 0.0186672803441373, + 0.0186672803441373, + 0.0186672803441373, + 0.018516397996184725, + 0.018516397996184725, + 0.018516397996184725, + 0.018641243669359635, + 0.018641243669359635, + 0.018641243669359635, + 0.018581482329561066, + 0.018581482329561066, + 0.018581482329561066, + 0.01392028150439728, + 0.01392028150439728, + 0.01392028150439728, + 0.01392028150439728, + 0.027841487986734137, + 0.027841487986734137, + 0.018460020675168682, + 0.018460020675168682, + 0.018460020675168682, + 0.01867730700178072, + 0.01867730700178072, + 0.01867730700178072, + 0.028588780987774953, + 0.028588780987774953, + 0.018283454002812505, + 0.018283454002812505, + 0.018283454002812505, + 0.02455506350088399, + 0.02455506350088399, + 0.02455506350088399, + 0.02455506350088399, + 0.018118704669177532, + 0.018118704669177532, + 0.018118704669177532, + 0.012909814744489267, + 0.012909814744489267, + 0.012909814744489267, + 0.012909814744489267, + 0.012955870508449152, + 0.012955870508449152, + 0.012955870508449152, + 0.012955870508449152, + 0.025617786479415372, + 0.025617786479415372, + 0.01710334767509873, + 0.01710334767509873, + 0.01710334767509873, + 0.01738610933534801, + 0.01738610933534801, + 0.01738610933534801, + 0.025520398980006576, + 0.025520398980006576, + 0.016854186678150047, + 0.016854186678150047, + 0.016854186678150047, + 0.01274280650250148, + 0.01274280650250148, + 0.01274280650250148, + 0.01274280650250148, + 0.02618332498241216, + 0.02618332498241216 + ], + [ + 0.07610312267206609, + 0.07610312267206609, + 0.07610312267206609, + 0.01501510600792244, + 0.01501510600792244, + 0.01501510600792244, + 0.01501510600792244, + 0.020073268989411492, + 0.020073268989411492, + 0.020073268989411492, + 0.020034901332110167, + 0.020034901332110167, + 0.020034901332110167, + 0.01501977325824555, + 0.01501977325824555, + 0.01501977325824555, + 0.01501977325824555, + 0.02973549149464816, + 0.02973549149464816, + 0.06096841796534136 + ] + ], + "generated_texts": [ + " the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed", + " dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit volupt", + " quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog.", + "piciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The", + " The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspic", + "atem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem", + " jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem", + "nis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis", + " error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over", + " voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde om" + ], + "errors": [ + "", + "", + "", + "", + "", + "", + "", + "", + "", + "" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-052659/report.md b/experiments/dsv4_p800_sglang/results/20260708-052659/report.md new file mode 100644 index 0000000..a845a16 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-052659/report.md @@ -0,0 +1,16 @@ +# Benchmark Report: dsv4_p800_sglang + +## Metadata + +- **Run ID**: 20260708-052659 +- **Timestamp**: 2026-07-08T05:26:59+00:00 +- **Chip/Accelerator**: Kunlun P800 XPU / kunlun_p800 +- **Engine/Backend**: sglang-xpu / sglang +- **Hardware**: 8x Kunlun P800 XPU +- **Model**: /data1/models/DeepSeek-V4-Flash-INT8 +- **Git Commit**: 4ef6543 + +## Results + +| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| diff --git a/experiments/dsv4_p800_sglang/results/20260708-052659/results.json b/experiments/dsv4_p800_sglang/results/20260708-052659/results.json new file mode 100644 index 0000000..ab11d28 --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-052659/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-052659", + "timestamp": "2026-07-08T05:26:59+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "4ef6543", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_p800_sglang/results/20260708-054332/results.json b/experiments/dsv4_p800_sglang/results/20260708-054332/results.json new file mode 100644 index 0000000..5c3f4eb --- /dev/null +++ b/experiments/dsv4_p800_sglang/results/20260708-054332/results.json @@ -0,0 +1,20 @@ +{ + "metadata": { + "experiment": "dsv4_p800_sglang", + "run_id": "20260708-054332", + "timestamp": "2026-07-08T05:43:32+00:00", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "backend": "sglang", + "engine": "sglang-xpu", + "hardware": "8x Kunlun P800 XPU", + "accelerator": "Kunlun P800 XPU", + "chip": "kunlun_p800", + "script": "experiments/dsv4_p800_sglang/run_bench.sh", + "env": "", + "git_commit": "4ef6543", + "git_dirty": "dirty", + "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" + }, + "config": {}, + "scenarios": [] +}