qqtang
d13f61f7b8
- Introduced `run_adaptive_concurrency_add16.sh` for adaptive concurrency testing with vLLM. - Added `run_bench.sh` to execute TP×DP matrix benchmarks for Hy3-FP8 on vLLM. - Created `start_vllm_docker.sh` to launch the vLLM server in a Docker container for specified TP and DP configurations. - Implemented `start_vllm_dp.sh` to manage starting the vLLM server, defaulting to Docker unless specified otherwise. - Enhanced logging and health check mechanisms for server readiness. - Included GPU monitoring and OOM detection in benchmarking scripts.
93 lines
1.5 KiB
JSON
93 lines
1.5 KiB
JSON
{
|
|
"comment": "Hy3 supports a 256K total context window. ISL >= 262144 is excluded because input plus output must remain within 262144 tokens.",
|
|
"mode": "Y",
|
|
"matrix": {
|
|
"1024": {
|
|
"128": "Y",
|
|
"256": "Y",
|
|
"512": "Y",
|
|
"1024": "Y",
|
|
"2048": "Y",
|
|
"4096": "Y"
|
|
},
|
|
"4096": {
|
|
"128": "Y",
|
|
"256": "Y",
|
|
"512": "Y",
|
|
"1024": "Y",
|
|
"2048": "Y",
|
|
"4096": "Y"
|
|
},
|
|
"8192": {
|
|
"128": "Y",
|
|
"256": "Y",
|
|
"512": "Y",
|
|
"1024": "Y",
|
|
"2048": "Y",
|
|
"4096": "Y"
|
|
},
|
|
"16384": {
|
|
"128": "Y",
|
|
"256": "Y",
|
|
"512": "Y",
|
|
"1024": "Y",
|
|
"2048": "Y",
|
|
"4096": "P"
|
|
},
|
|
"32768": {
|
|
"128": "Y",
|
|
"256": "Y",
|
|
"512": "Y",
|
|
"1024": "Y",
|
|
"2048": "Y",
|
|
"4096": "P"
|
|
},
|
|
"65536": {
|
|
"128": "Y",
|
|
"256": "Y",
|
|
"512": "Y",
|
|
"1024": "Y",
|
|
"2048": "P",
|
|
"4096": "N"
|
|
},
|
|
"131072": {
|
|
"128": "Y",
|
|
"256": "Y",
|
|
"512": "Y",
|
|
"1024": "P",
|
|
"2048": "N",
|
|
"4096": "N"
|
|
}
|
|
},
|
|
"concurrency": {
|
|
"1024": {
|
|
"low": 1,
|
|
"high": 128
|
|
},
|
|
"4096": {
|
|
"low": 1,
|
|
"high": 64
|
|
},
|
|
"8192": {
|
|
"low": 1,
|
|
"high": 64
|
|
},
|
|
"16384": {
|
|
"low": 1,
|
|
"high": 32
|
|
},
|
|
"32768": {
|
|
"low": 1,
|
|
"high": 16
|
|
},
|
|
"65536": {
|
|
"low": 1,
|
|
"high": 8
|
|
},
|
|
"131072": {
|
|
"low": 1,
|
|
"high": 4
|
|
}
|
|
}
|
|
}
|