{
  "metadata": {
    "experiment": "dsv4_h200_max_context_length",
    "run_id": "20260708-143534",
    "timestamp": "2026-07-08T14:35:34+00:00",
    "model": "/data/models/DeepSeek-V4-Flash",
    "backend": "sglang+vllm",
    "engine": "sglang+vllm",
    "hardware": "8x NVIDIA H200 143GB",
    "accelerator": "NVIDIA H200",
    "chip": "nvidia_h200",
    "script": "experiments/dsv4_h200_max_context_length/run_bench.sh",
    "env": "/data/user1/yy/envs/sglang",
    "git_commit": "ef65885",
    "git_dirty": "dirty",
    "description": "H200 max context length exploration for DeepSeek-V4-Flash"
  },
  "config": {
    "tp": 8,
    "cuda_visible_devices": "0,1,2,3,4,5,6,7",
    "output_len": 1,
    "num_prompts": 1,
    "max_concurrency": 1,
    "request_rate": 10000,
    "context_pad": 1024
  },
  "scenarios": [
    {
      "backend": "sglang",
      "target_len": 65536,
      "max_model_len": 66560,
      "success": true,
      "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 66560 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
      "metrics": {
        "success_count": 1,
        "total_count": 1,
        "duration_s": 0.5241995020041941,
        "request_throughput": 1.9076706409995772,
        "input_token_throughput": 125021.10312854829,
        "output_token_throughput": 1.9076706409995772,
        "total_input_tokens": 65536,
        "total_output_tokens": 1,
        "e2e_ms": {
          "mean": 521.2510440032929,
          "p50": 521.2510440032929,
          "p90": 521.2510440032929,
          "p95": 521.2510440032929,
          "p99": 521.2510440032929
        },
        "ttft_ms": {
          "mean": 521.2282620050246,
          "p50": 521.2282620050246,
          "p90": 521.2282620050246,
          "p95": 521.2282620050246,
          "p99": 521.2282620050246
        },
        "tpot_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        },
        "itl_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        }
      },
      "error": null
    },
    {
      "backend": "sglang",
      "target_len": 131072,
      "max_model_len": 132096,
      "success": true,
      "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 132096 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
      "metrics": {
        "success_count": 1,
        "total_count": 1,
        "duration_s": 0.6736304189980729,
        "request_throughput": 1.484493532057763,
        "input_token_throughput": 194575.53623387511,
        "output_token_throughput": 1.484493532057763,
        "total_input_tokens": 131072,
        "total_output_tokens": 1,
        "e2e_ms": {
          "mean": 668.692392995581,
          "p50": 668.692392995581,
          "p90": 668.692392995581,
          "p95": 668.692392995581,
          "p99": 668.692392995581
        },
        "ttft_ms": {
          "mean": 668.666598998243,
          "p50": 668.666598998243,
          "p90": 668.666598998243,
          "p95": 668.666598998243,
          "p99": 668.666598998243
        },
        "tpot_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        },
        "itl_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        }
      },
      "error": null
    },
    {
      "backend": "sglang",
      "target_len": 262144,
      "max_model_len": 263168,
      "success": true,
      "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 263168 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
      "metrics": {
        "success_count": 1,
        "total_count": 1,
        "duration_s": 1.235619597006007,
        "request_throughput": 0.8093105697118029,
        "input_token_throughput": 212155.90998653087,
        "output_token_throughput": 0.8093105697118029,
        "total_input_tokens": 262144,
        "total_output_tokens": 1,
        "e2e_ms": {
          "mean": 1232.6548010023544,
          "p50": 1232.6548010023544,
          "p90": 1232.6548010023544,
          "p95": 1232.6548010023544,
          "p99": 1232.6548010023544
        },
        "ttft_ms": {
          "mean": 1232.628379992093,
          "p50": 1232.628379992093,
          "p90": 1232.628379992093,
          "p95": 1232.628379992093,
          "p99": 1232.628379992093
        },
        "tpot_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        },
        "itl_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        }
      },
      "error": null
    },
    {
      "backend": "sglang",
      "target_len": 393216,
      "max_model_len": 394240,
      "success": true,
      "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 394240 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
      "metrics": {
        "success_count": 1,
        "total_count": 1,
        "duration_s": 1.4997316709923325,
        "request_throughput": 0.6667859453406932,
        "input_token_throughput": 262190.90228308603,
        "output_token_throughput": 0.6667859453406932,
        "total_input_tokens": 393216,
        "total_output_tokens": 1,
        "e2e_ms": {
          "mean": 1496.5319500042824,
          "p50": 1496.5319500042824,
          "p90": 1496.5319500042824,
          "p95": 1496.5319500042824,
          "p99": 1496.5319500042824
        },
        "ttft_ms": {
          "mean": 1496.4891230047215,
          "p50": 1496.4891230047215,
          "p90": 1496.4891230047215,
          "p95": 1496.4891230047215,
          "p99": 1496.4891230047215
        },
        "tpot_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        },
        "itl_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        }
      },
      "error": null
    },
    {
      "backend": "sglang",
      "target_len": 524288,
      "max_model_len": 525312,
      "success": true,
      "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 525312 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
      "metrics": {
        "success_count": 1,
        "total_count": 1,
        "duration_s": 1.7669223830016563,
        "request_throughput": 0.5659558165204717,
        "input_token_throughput": 296723.84313188505,
        "output_token_throughput": 0.5659558165204717,
        "total_input_tokens": 524288,
        "total_output_tokens": 1,
        "e2e_ms": {
          "mean": 1764.228992004064,
          "p50": 1764.228992004064,
          "p90": 1764.228992004064,
          "p95": 1764.228992004064,
          "p99": 1764.228992004064
        },
        "ttft_ms": {
          "mean": 1764.184692001436,
          "p50": 1764.184692001436,
          "p90": 1764.184692001436,
          "p95": 1764.184692001436,
          "p99": 1764.184692001436
        },
        "tpot_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        },
        "itl_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        }
      },
      "error": null
    },
    {
      "backend": "sglang",
      "target_len": 655360,
      "max_model_len": 656384,
      "success": true,
      "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 656384 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
      "metrics": {
        "success_count": 1,
        "total_count": 1,
        "duration_s": 2.1324244450079277,
        "request_throughput": 0.4689497920271132,
        "input_token_throughput": 307330.93570288894,
        "output_token_throughput": 0.4689497920271132,
        "total_input_tokens": 655360,
        "total_output_tokens": 1,
        "e2e_ms": {
          "mean": 2129.2401870014146,
          "p50": 2129.2401870014146,
          "p90": 2129.2401870014146,
          "p95": 2129.2401870014146,
          "p99": 2129.2401870014146
        },
        "ttft_ms": {
          "mean": 2129.1923570097424,
          "p50": 2129.1923570097424,
          "p90": 2129.1923570097424,
          "p95": 2129.1923570097424,
          "p99": 2129.1923570097424
        },
        "tpot_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        },
        "itl_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        }
      },
      "error": null
    },
    {
      "backend": "sglang",
      "target_len": 786432,
      "max_model_len": 787456,
      "success": true,
      "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 787456 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
      "metrics": {
        "success_count": 1,
        "total_count": 1,
        "duration_s": 2.4429005489946576,
        "request_throughput": 0.409349451581865,
        "input_token_throughput": 321925.50790642924,
        "output_token_throughput": 0.409349451581865,
        "total_input_tokens": 786432,
        "total_output_tokens": 1,
        "e2e_ms": {
          "mean": 2440.3524239896797,
          "p50": 2440.3524239896797,
          "p90": 2440.3524239896797,
          "p95": 2440.3524239896797,
          "p99": 2440.3524239896797
        },
        "ttft_ms": {
          "mean": 2440.341288995114,
          "p50": 2440.341288995114,
          "p90": 2440.341288995114,
          "p95": 2440.341288995114,
          "p99": 2440.341288995114
        },
        "tpot_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        },
        "itl_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        }
      },
      "error": null
    },
    {
      "backend": "sglang",
      "target_len": 917504,
      "max_model_len": 918528,
      "success": true,
      "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 918528 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
      "metrics": {
        "success_count": 1,
        "total_count": 1,
        "duration_s": 3.152248802012764,
        "request_throughput": 0.31723384250681075,
        "input_token_throughput": 291063.3194353689,
        "output_token_throughput": 0.31723384250681075,
        "total_input_tokens": 917504,
        "total_output_tokens": 1,
        "e2e_ms": {
          "mean": 3149.605149999843,
          "p50": 3149.605149999843,
          "p90": 3149.605149999843,
          "p95": 3149.605149999843,
          "p99": 3149.605149999843
        },
        "ttft_ms": {
          "mean": 3149.579901000834,
          "p50": 3149.579901000834,
          "p90": 3149.579901000834,
          "p95": 3149.579901000834,
          "p99": 3149.579901000834
        },
        "tpot_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        },
        "itl_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        }
      },
      "error": null
    },
    {
      "backend": "sglang",
      "target_len": 1048576,
      "max_model_len": 1049600,
      "success": false,
      "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 1049600 --max-running-requests 1 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30011",
      "metrics": null,
      "error": "server failed to start"
    },
    {
      "backend": "vllm",
      "target_len": 65536,
      "max_model_len": 66560,
      "success": true,
      "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 66560 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
      "metrics": {
        "success_count": 1,
        "total_count": 1,
        "duration_s": 0.5968579519976629,
        "request_throughput": 1.675440524253777,
        "input_token_throughput": 109801.67019749554,
        "output_token_throughput": 1.675440524253777,
        "total_input_tokens": 65536,
        "total_output_tokens": 1,
        "e2e_ms": {
          "mean": 594.5277150021866,
          "p50": 594.5277150021866,
          "p90": 594.5277150021866,
          "p95": 594.5277150021866,
          "p99": 594.5277150021866
        },
        "ttft_ms": {
          "mean": 594.4739139958983,
          "p50": 594.4739139958983,
          "p90": 594.4739139958983,
          "p95": 594.4739139958983,
          "p99": 594.4739139958983
        },
        "tpot_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        },
        "itl_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        }
      },
      "error": null
    },
    {
      "backend": "vllm",
      "target_len": 131072,
      "max_model_len": 132096,
      "success": true,
      "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 132096 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
      "metrics": {
        "success_count": 1,
        "total_count": 1,
        "duration_s": 0.8928074569994351,
        "request_throughput": 1.1200623294084255,
        "input_token_throughput": 146808.80964022115,
        "output_token_throughput": 1.1200623294084255,
        "total_input_tokens": 131072,
        "total_output_tokens": 1,
        "e2e_ms": {
          "mean": 889.5100749941776,
          "p50": 889.5100749941776,
          "p90": 889.5100749941776,
          "p95": 889.5100749941776,
          "p99": 889.5100749941776
        },
        "ttft_ms": {
          "mean": 889.4585100060795,
          "p50": 889.4585100060795,
          "p90": 889.4585100060795,
          "p95": 889.4585100060795,
          "p99": 889.4585100060795
        },
        "tpot_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        },
        "itl_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        }
      },
      "error": null
    },
    {
      "backend": "vllm",
      "target_len": 262144,
      "max_model_len": 263168,
      "success": true,
      "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 263168 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
      "metrics": {
        "success_count": 1,
        "total_count": 1,
        "duration_s": 1.1815237270056969,
        "request_throughput": 0.8463647213706597,
        "input_token_throughput": 221869.4335189902,
        "output_token_throughput": 0.8463647213706597,
        "total_input_tokens": 262144,
        "total_output_tokens": 1,
        "e2e_ms": {
          "mean": 1178.564740010188,
          "p50": 1178.564740010188,
          "p90": 1178.564740010188,
          "p95": 1178.564740010188,
          "p99": 1178.564740010188
        },
        "ttft_ms": {
          "mean": 1178.5299420007505,
          "p50": 1178.5299420007505,
          "p90": 1178.5299420007505,
          "p95": 1178.5299420007505,
          "p99": 1178.5299420007505
        },
        "tpot_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        },
        "itl_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        }
      },
      "error": null
    },
    {
      "backend": "vllm",
      "target_len": 393216,
      "max_model_len": 394240,
      "success": true,
      "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 394240 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
      "metrics": {
        "success_count": 1,
        "total_count": 1,
        "duration_s": 1.3780717330082553,
        "request_throughput": 0.725651630497532,
        "input_token_throughput": 285337.83153771755,
        "output_token_throughput": 0.725651630497532,
        "total_input_tokens": 393216,
        "total_output_tokens": 1,
        "e2e_ms": {
          "mean": 1371.242612003698,
          "p50": 1371.242612003698,
          "p90": 1371.242612003698,
          "p95": 1371.242612003698,
          "p99": 1371.242612003698
        },
        "ttft_ms": {
          "mean": 1371.1869800026761,
          "p50": 1371.1869800026761,
          "p90": 1371.1869800026761,
          "p95": 1371.1869800026761,
          "p99": 1371.1869800026761
        },
        "tpot_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        },
        "itl_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        }
      },
      "error": null
    },
    {
      "backend": "vllm",
      "target_len": 524288,
      "max_model_len": 525312,
      "success": true,
      "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 525312 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
      "metrics": {
        "success_count": 1,
        "total_count": 1,
        "duration_s": 2.587135097011924,
        "request_throughput": 0.3865279401740462,
        "input_token_throughput": 202651.96069797032,
        "output_token_throughput": 0.3865279401740462,
        "total_input_tokens": 524288,
        "total_output_tokens": 1,
        "e2e_ms": {
          "mean": 2582.903844988323,
          "p50": 2582.903844988323,
          "p90": 2582.903844988323,
          "p95": 2582.903844988323,
          "p99": 2582.903844988323
        },
        "ttft_ms": {
          "mean": 2582.850620994577,
          "p50": 2582.850620994577,
          "p90": 2582.850620994577,
          "p95": 2582.850620994577,
          "p99": 2582.850620994577
        },
        "tpot_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        },
        "itl_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        }
      },
      "error": null
    },
    {
      "backend": "vllm",
      "target_len": 655360,
      "max_model_len": 656384,
      "success": true,
      "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 656384 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
      "metrics": {
        "success_count": 1,
        "total_count": 1,
        "duration_s": 2.206327235006029,
        "request_throughput": 0.453241923561383,
        "input_token_throughput": 297036.62702518795,
        "output_token_throughput": 0.453241923561383,
        "total_input_tokens": 655360,
        "total_output_tokens": 1,
        "e2e_ms": {
          "mean": 2203.5920649941545,
          "p50": 2203.5920649941545,
          "p90": 2203.5920649941545,
          "p95": 2203.5920649941545,
          "p99": 2203.5920649941545
        },
        "ttft_ms": {
          "mean": 2203.5506129905116,
          "p50": 2203.5506129905116,
          "p90": 2203.5506129905116,
          "p95": 2203.5506129905116,
          "p99": 2203.5506129905116
        },
        "tpot_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        },
        "itl_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        }
      },
      "error": null
    },
    {
      "backend": "vllm",
      "target_len": 786432,
      "max_model_len": 787456,
      "success": true,
      "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 787456 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
      "metrics": {
        "success_count": 1,
        "total_count": 1,
        "duration_s": 3.8894555929873604,
        "request_throughput": 0.25710539073977023,
        "input_token_throughput": 202195.90665025898,
        "output_token_throughput": 0.25710539073977023,
        "total_input_tokens": 786432,
        "total_output_tokens": 1,
        "e2e_ms": {
          "mean": 3886.5709000092465,
          "p50": 3886.5709000092465,
          "p90": 3886.5709000092465,
          "p95": 3886.5709000092465,
          "p99": 3886.5709000092465
        },
        "ttft_ms": {
          "mean": 3886.5172150108265,
          "p50": 3886.5172150108265,
          "p90": 3886.5172150108265,
          "p95": 3886.5172150108265,
          "p99": 3886.5172150108265
        },
        "tpot_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        },
        "itl_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        }
      },
      "error": null
    },
    {
      "backend": "vllm",
      "target_len": 917504,
      "max_model_len": 918528,
      "success": true,
      "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 918528 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
      "metrics": {
        "success_count": 1,
        "total_count": 1,
        "duration_s": 3.7180563760048244,
        "request_throughput": 0.26895772921940825,
        "input_token_throughput": 246769.79238972397,
        "output_token_throughput": 0.26895772921940825,
        "total_input_tokens": 917504,
        "total_output_tokens": 1,
        "e2e_ms": {
          "mean": 3715.635304004536,
          "p50": 3715.635304004536,
          "p90": 3715.635304004536,
          "p95": 3715.635304004536,
          "p99": 3715.635304004536
        },
        "ttft_ms": {
          "mean": 3715.6081900029676,
          "p50": 3715.6081900029676,
          "p90": 3715.6081900029676,
          "p95": 3715.6081900029676,
          "p99": 3715.6081900029676
        },
        "tpot_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        },
        "itl_ms": {
          "mean": 0.0,
          "p50": 0.0,
          "p90": 0.0,
          "p95": 0.0,
          "p99": 0.0
        }
      },
      "error": null
    },
    {
      "backend": "vllm",
      "target_len": 1048576,
      "max_model_len": 1049600,
      "success": false,
      "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 1049600 --max-num-seqs 1 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30012",
      "metrics": null,
      "error": "server failed to start"
    }
  ]
}