From 080e095ecef1084e5193ac89e7c5f3b6bbf7387d Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Tue, 21 Jul 2026 09:44:41 +0000 Subject: [PATCH] =?UTF-8?q?chore(repo):=20=E7=98=A6=E8=BA=AB=E8=A7=84?= =?UTF-8?q?=E8=8C=83=20-=20=E5=BF=BD=E7=95=A5=E7=BC=96=E8=AF=91=E4=BA=A7?= =?UTF-8?q?=E7=89=A9/JIT=E7=BC=93=E5=AD=98/raw=5Foutputs=EF=BC=8C=E5=8F=96?= =?UTF-8?q?=E6=B6=88=E8=B7=9F=E8=B8=AA618=E4=B8=AA=E5=9E=83=E5=9C=BE?= =?UTF-8?q?=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - .gitignore 新增规范：实验目录只提交 代码 + report.md + results.json(小体量汇总) - 忽略: *.o/*.so/*.ninja*/*.cubin, sglang_sm120_cache/, vllm_sm120_cache/, *_sm120_cache/, experiments/**/runtime/, bench-output/, experiments/**/raw_outputs/ - 修正原 experiments/*/runtime/ 规则过窄(只匹配一层) -> experiments/**/runtime/ - git rm --cached 取消跟踪 618 个已入库的缓存/产物/原始日志(文件保留在本地工作区) - 当前 HEAD 减少约 83MB 跟踪体积; 历史体积需另做 filter-branch 重写(本次不做) - results.json 保留(28个共0.99MB, compare.py/文档依赖) Co-Authored-By: Claude Fable 5 --- .gitignore | 29 +- .../device_kernel.cu | 424 -- .../host_kernel.cu | 3359 ---------------- .../params.pkl | Bin 2200 -> 0 bytes .../device_kernel.cu | 424 -- .../host_kernel.cu | 3359 ---------------- .../params.pkl | Bin 2200 -> 0 bytes .../device_kernel.cu | 424 -- .../host_kernel.cu | 3359 ---------------- .../params.pkl | Bin 2200 -> 0 bytes .../device_kernel.cu | 424 -- .../host_kernel.cu | 3359 ---------------- .../params.pkl | Bin 2200 -> 0 bytes .../device_kernel.cu | 424 -- .../host_kernel.cu | 3359 ---------------- .../params.pkl | Bin 2200 -> 0 bytes .../device_kernel.cu | 74 - .../host_kernel.cu | 2176 ----------- .../params.pkl | Bin 1725 -> 0 bytes .../device_kernel.cu | 424 -- .../host_kernel.cu | 3359 ---------------- .../params.pkl | Bin 2200 -> 0 bytes .../device_kernel.cu | 424 -- .../host_kernel.cu | 3359 ---------------- .../params.pkl | Bin 2200 -> 0 bytes .../device_kernel.cu | 424 -- .../host_kernel.cu | 3359 ---------------- .../params.pkl | Bin 2200 -> 0 bytes .../device_kernel.cu | 424 -- .../host_kernel.cu | 3359 ---------------- .../params.pkl | Bin 2200 -> 0 bytes .../device_kernel.cu | 424 -- .../host_kernel.cu | 3359 ---------------- .../params.pkl | Bin 2200 -> 0 bytes .../device_kernel.cu | 424 -- .../host_kernel.cu | 3359 ---------------- .../params.pkl | Bin 2200 -> 0 bytes .../device_kernel.cu | 420 -- .../host_kernel.cu | 3289 ---------------- .../params.pkl | Bin 2200 -> 0 bytes .../device_kernel.cu | 424 -- .../host_kernel.cu | 3359 ---------------- .../params.pkl | Bin 2200 -> 0 bytes .../device_kernel.cu | 104 - .../host_kernel.cu | 3435 ----------------- .../params.pkl | Bin 2862 -> 0 bytes .../device_kernel.cu | 102 - .../host_kernel.cu | 3435 ----------------- .../params.pkl | Bin 2862 -> 0 bytes .../device_kernel.cu | 190 - .../host_kernel.cu | 1828 --------- .../params.pkl | Bin 1000 -> 0 bytes .../device_kernel.cu | 424 -- .../host_kernel.cu | 3359 ---------------- .../params.pkl | Bin 2200 -> 0 bytes .../device_kernel.cu | 424 -- .../host_kernel.cu | 3359 ---------------- .../params.pkl | Bin 2200 -> 0 bytes .../device_kernel.cu | 424 -- .../host_kernel.cu | 3359 ---------------- .../params.pkl | Bin 2200 -> 0 bytes 61 files changed, 27 insertions(+), 71800 deletions(-) delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/0f368e85ad91dc38d1eca212e6350c2e26e5f453a6c404b9ceb10dca001fe742/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/0f368e85ad91dc38d1eca212e6350c2e26e5f453a6c404b9ceb10dca001fe742/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/0f368e85ad91dc38d1eca212e6350c2e26e5f453a6c404b9ceb10dca001fe742/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/147eb04270024c36acaf2922c18e38bbdfc222673231f206ebd108766e937d6c/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/147eb04270024c36acaf2922c18e38bbdfc222673231f206ebd108766e937d6c/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/147eb04270024c36acaf2922c18e38bbdfc222673231f206ebd108766e937d6c/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1eae71a46e812d68441e1ebddd96e5dbecfaeec6b4f8ed7749c4f21be97720e8/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1eae71a46e812d68441e1ebddd96e5dbecfaeec6b4f8ed7749c4f21be97720e8/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1eae71a46e812d68441e1ebddd96e5dbecfaeec6b4f8ed7749c4f21be97720e8/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f417b351704372ae9697b314cec4913d9e69469d246e2c9d2c8400a16396625/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f417b351704372ae9697b314cec4913d9e69469d246e2c9d2c8400a16396625/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f417b351704372ae9697b314cec4913d9e69469d246e2c9d2c8400a16396625/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f58ab0397f1bec57393c246b40519995302f81a4f075621dbc500f561f77d46/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f58ab0397f1bec57393c246b40519995302f81a4f075621dbc500f561f77d46/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f58ab0397f1bec57393c246b40519995302f81a4f075621dbc500f561f77d46/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2271822608992f393db54113760d3441caf5e37b0ed41e8e367fb0b64d28f248/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2271822608992f393db54113760d3441caf5e37b0ed41e8e367fb0b64d28f248/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2271822608992f393db54113760d3441caf5e37b0ed41e8e367fb0b64d28f248/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/25b1b2d19c0004c3d858fe560874c170531adddd2ba8ea1ea0a4020fd5aeb5a3/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/25b1b2d19c0004c3d858fe560874c170531adddd2ba8ea1ea0a4020fd5aeb5a3/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/25b1b2d19c0004c3d858fe560874c170531adddd2ba8ea1ea0a4020fd5aeb5a3/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2f65ba228083b031f9f8b7020d4a171fc67be94c7b476fa6fbe648831cff6878/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2f65ba228083b031f9f8b7020d4a171fc67be94c7b476fa6fbe648831cff6878/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2f65ba228083b031f9f8b7020d4a171fc67be94c7b476fa6fbe648831cff6878/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/34141341ae4687b7d5cb99f48bcdb971fae6efe8bba5eed695620f0e86b8bdd4/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/34141341ae4687b7d5cb99f48bcdb971fae6efe8bba5eed695620f0e86b8bdd4/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/34141341ae4687b7d5cb99f48bcdb971fae6efe8bba5eed695620f0e86b8bdd4/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/455ef1b63b3a592f4741c938fcfb8d01f90e8269880307cffbf4438ed668de0b/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/455ef1b63b3a592f4741c938fcfb8d01f90e8269880307cffbf4438ed668de0b/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/455ef1b63b3a592f4741c938fcfb8d01f90e8269880307cffbf4438ed668de0b/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/47c189497f8ff487a170b06db17717920a83856d30485b87094ce9d42596885b/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/47c189497f8ff487a170b06db17717920a83856d30485b87094ce9d42596885b/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/47c189497f8ff487a170b06db17717920a83856d30485b87094ce9d42596885b/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/596345ac7ba68225bdd1b9b6abe5e4fbb63f7c57ec2523668ca14b62460b457a/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/596345ac7ba68225bdd1b9b6abe5e4fbb63f7c57ec2523668ca14b62460b457a/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/596345ac7ba68225bdd1b9b6abe5e4fbb63f7c57ec2523668ca14b62460b457a/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/690cfe1726579f52e2028c161d3001b9f82af97555a40aab825d6e0140ad4e12/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/690cfe1726579f52e2028c161d3001b9f82af97555a40aab825d6e0140ad4e12/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/690cfe1726579f52e2028c161d3001b9f82af97555a40aab825d6e0140ad4e12/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/8ac26e5a6bf2572ea507280b0cfe38c49999af7cb683ccecde6888c37a6e6f3a/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/8ac26e5a6bf2572ea507280b0cfe38c49999af7cb683ccecde6888c37a6e6f3a/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/8ac26e5a6bf2572ea507280b0cfe38c49999af7cb683ccecde6888c37a6e6f3a/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/94b96c76ed4b18c5a1404f5c71d4cc9c8be8d3d582285c754714502150b21353/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/94b96c76ed4b18c5a1404f5c71d4cc9c8be8d3d582285c754714502150b21353/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/94b96c76ed4b18c5a1404f5c71d4cc9c8be8d3d582285c754714502150b21353/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/96e9b9b5cf258ab70fb96c06c4ba12fa70819bfb1b8c147e903abf420053524b/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/96e9b9b5cf258ab70fb96c06c4ba12fa70819bfb1b8c147e903abf420053524b/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/96e9b9b5cf258ab70fb96c06c4ba12fa70819bfb1b8c147e903abf420053524b/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/bc130d0880c8dc66c0a86cfbe52a5accea0d7723ef84286eda1809080514e34c/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/bc130d0880c8dc66c0a86cfbe52a5accea0d7723ef84286eda1809080514e34c/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/bc130d0880c8dc66c0a86cfbe52a5accea0d7723ef84286eda1809080514e34c/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/c0ef72219d5280025634d12a15beae51ea48dcaded4066ba7f411defa4b60bff/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/c0ef72219d5280025634d12a15beae51ea48dcaded4066ba7f411defa4b60bff/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/c0ef72219d5280025634d12a15beae51ea48dcaded4066ba7f411defa4b60bff/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/e8bfe015e13b0beed5fbf1df07913214e93cd08d927e1e810f022bb8d28739b0/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/e8bfe015e13b0beed5fbf1df07913214e93cd08d927e1e810f022bb8d28739b0/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/e8bfe015e13b0beed5fbf1df07913214e93cd08d927e1e810f022bb8d28739b0/params.pkl delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/ee551f0fd55a5f344ee5ec2ee5aa79c7656eb4c9efbd0c4a7c2d3107c1e82478/device_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/ee551f0fd55a5f344ee5ec2ee5aa79c7656eb4c9efbd0c4a7c2d3107c1e82478/host_kernel.cu delete mode 100644 experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/ee551f0fd55a5f344ee5ec2ee5aa79c7656eb4c9efbd0c4a7c2d3107c1e82478/params.pkl diff --git a/.gitignore b/.gitignore index bafac7f..263811e 100644 --- a/.gitignore +++ b/.gitignore @@ -42,7 +42,7 @@ experiments/*/results/*/*.tsv experiments/*/results/*/*/*.tsv # 运行时目录（pid 文件、缓存、临时文件） -experiments/*/runtime/ +experiments/**/runtime/ # 无关项目 loomeval_yy/ @@ -51,4 +51,29 @@ loomeval_yy/ dsv4_dspark_h20_sglang_tp_dp_matrix dsv4_h20_sglang_tp_dp_matrix -glm_old_h20_vllm_tp_dp_matrix \ No newline at end of file +glm_old_h20_vllm_tp_dp_matrix +# === 仓库瘦身规范（2026-07）=== +# 约定：实验目录只提交 代码(config.env/*.sh/*.py) + report.md + results.json(小体量汇总)。 +# 不提交：逐请求原始日志(raw_outputs)、JIT/算子缓存、编译产物、一次性 bench 输出。 +# 历史中 raw_outputs/*.jsonl 是仓库膨胀主因（~1.3GB）；本规范从源头不再入库。 +# results.json 须保持小体量，禁止嵌入 raw_requests（否则单独走 raw_outputs）。 + +# 编译产物（每次构建重新生成） +*.o +*.so +*.ninja +*.ninja_deps +*.ninja_log +*.cubin +build/ + +# JIT / 算子缓存（每次跑重新生成） +sglang_sm120_cache/ +vllm_sm120_cache/ +*_sm120_cache/ + +# 一次性 bench 输出 +bench-output/ + +# 逐请求原始日志（体积大；汇总见 results.json / report.md） +experiments/**/raw_outputs/ diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/0f368e85ad91dc38d1eca212e6350c2e26e5f453a6c404b9ceb10dca001fe742/device_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/0f368e85ad91dc38d1eca212e6350c2e26e5f453a6c404b9ceb10dca001fe742/device_kernel.cu deleted file mode 100644 index ce32651..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/0f368e85ad91dc38d1eca212e6350c2e26e5f453a6c404b9ceb10dca001fe742/device_kernel.cu +++ /dev/null @@ -1,424 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float mixes[1]; - float rms[1]; - float cm[1]; - float row_max[1]; - float row_sum[1]; - float col_sum[1]; - float sumsq_per_pos[16]; - float sumsq[1]; - float rsqrt_norm[1]; - float xl[64]; - float ol[16]; - bfloat16_t xs_local_cast[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t xs_local_cast_2[8]; - float w_local[16]; - float ol_1[16]; - bfloat16_t w_shared_local_cast_3[8]; - bfloat16_t output_shared_local_cast_4[8]; - bfloat16_t layer_input_local_cast_5[8]; - bfloat16_t w_shared_local_cast_6[8]; - bfloat16_t output_shared_local_cast_7[8]; - bfloat16_t layer_input_local_cast_8[8]; - bfloat16_t w_shared_local_cast_9[8]; - bfloat16_t output_shared_local_cast_10[8]; - bfloat16_t layer_input_local_cast_11[8]; - mixes[0] = 0x0p+0f/*0.000000e+00*/; - rms[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - for (int i_split = 0; i_split < 64; ++i_split) { - rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]); - } - rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - mixes[0] = 0x0p+0f/*0.000000e+00*/; - for (int i_split_1 = 0; i_split_1 < 64; ++i_split_1) { - mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]); - } - mixes[0] = (mixes[0] * rms[0]); - if ((((int)threadIdx.x) / 24) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0]; - } - __syncthreads(); - if (((int)threadIdx.x) < 32) { - if (((int)threadIdx.x) < 4) { - post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/); - } - cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]); - row_max[0] = -CUDART_INF_F; - row_max[0] = max(row_max[0], cm[0]); - row_max[0] = tl::AllReduce>::run(row_max[0]); - cm[0] = expf((cm[0] - row_max[0])); - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - for (int __1 = 0; __1 < 19; ++__1) { - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - } - if ((((int)threadIdx.x) >> 4) == 0) { - comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0]; - } - } else { - if (((int)threadIdx.x) < 36) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - #pragma unroll - for (int i = 0; i < 4; ++i) { - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_2 = 0; i_2 < 8; ++i_2) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_4 = 0; i_4 < 8; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - tl::__sync_thread_partial<3, 64>(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)]; - #pragma unroll - for (int i_6 = 0; i_6 < 16; ++i_6) { - ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)])); - } - } - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - float4 __3; - float4 v__1 = *(float4*)(sumsq_per_pos + (i_7 * 4)); - float4 __4; - float4 v__2 = *(float4*)(ol + (i_7 * 4)); - __4.x = (v__2.x*v__2.x); - __4.y = (v__2.y*v__2.y); - __4.z = (v__2.z*v__2.z); - __4.w = (v__2.w*v__2.w); - __3.x = (v__1.x+__4.x); - __3.y = (v__1.y+__4.y); - __3.z = (v__1.z+__4.z); - __3.w = (v__1.w+__4.w); - *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3; - uint2 __5; - float4 v__3 = *(float4*)(ol + (i_7 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __5; - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_8 = 0; i_8 < 8; ++i_8) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __6; - uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]); - *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __6; - } - } - #pragma unroll - for (int i_9 = 0; i_9 < 4; ++i_9) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)]; - #pragma unroll - for (int i_10 = 0; i_10 < 16; ++i_10) { - ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)])); - } - } - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - float4 __7; - float4 v__5 = *(float4*)(sumsq_per_pos + (i_11 * 4)); - float4 __8; - float4 v__6 = *(float4*)(ol + (i_11 * 4)); - __8.x = (v__6.x*v__6.x); - __8.y = (v__6.y*v__6.y); - __8.z = (v__6.z*v__6.z); - __8.w = (v__6.w*v__6.w); - __7.x = (v__5.x+__8.x); - __7.y = (v__5.y+__8.y); - __7.z = (v__5.z+__8.z); - __7.w = (v__5.w+__8.w); - *(float4*)(sumsq_per_pos + (i_11 * 4)) = __7; - uint2 __9; - float4 v__7 = *(float4*)(ol + (i_11 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__9))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__9))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __9; - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_12 = 0; i_12 < 8; ++i_12) { - *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __10; - uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4)); - ((float2*)(&__10))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]); - ((float2*)(&__10))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]); - *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __10; - } - } - #pragma unroll - for (int i_13 = 0; i_13 < 4; ++i_13) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)]; - #pragma unroll - for (int i_14 = 0; i_14 < 16; ++i_14) { - ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)])); - } - } - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - float4 __11; - float4 v__9 = *(float4*)(sumsq_per_pos + (i_15 * 4)); - float4 __12; - float4 v__10 = *(float4*)(ol + (i_15 * 4)); - __12.x = (v__10.x*v__10.x); - __12.y = (v__10.y*v__10.y); - __12.z = (v__10.z*v__10.z); - __12.w = (v__10.w*v__10.w); - __11.x = (v__9.x+__12.x); - __11.y = (v__9.y+__12.y); - __11.z = (v__9.z+__12.z); - __11.w = (v__9.w+__12.w); - *(float4*)(sumsq_per_pos + (i_15 * 4)) = __11; - uint2 __13; - float4 v__11 = *(float4*)(ol + (i_15 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__13))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__13))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __13; - } - sumsq[0] = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int rv = 0; rv < 16; ++rv) { - sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]); - } - tl::__sync_thread_partial<3, 64>(); - sumsq[0] = tl::AllReduce>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192]))); - rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_17 = 0; i_17 < 2; ++i_17) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)]))); - } - tl::cp_async_commit(); - for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_18 = 0; i_18 < 2; ++i_18) { - *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_3 = 0; vec_3 < 2; ++vec_3) { - float4 __14; - uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4)); - ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]); - ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]); - *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __14; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_19 = 0; i_19 < 2; ++i_19) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)]))); - } - tl::cp_async_commit(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_20 = 0; i_20 < 2; ++i_20) { - *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984)); - for (int vec_4 = 0; vec_4 < 2; ++vec_4) { - float4 __15; - float4 __16; - float4 __17; - uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4)); - ((float2*)(&__17))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]); - ((float2*)(&__17))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]); - float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __16.x = (__17.x*v__14.x); - __16.y = (__17.y*v__14.y); - __16.z = (__17.z*v__14.z); - __16.w = (__17.w*v__14.w); - float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4))); - __15.x = (__16.x*v__15.x); - __15.y = (__16.y*v__15.y); - __15.z = (__16.z*v__15.z); - __15.w = (__16.w*v__15.w); - *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __15; - } - } - #pragma unroll - for (int i_21 = 0; i_21 < 2; ++i_21) { - for (int vec_5 = 0; vec_5 < 2; ++vec_5) { - uint2 __18; - float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__18))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__18))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]); - *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __18; - } - *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0); - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_22 = 0; i_22 < 2; ++i_22) { - *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_6 = 0; vec_6 < 2; ++vec_6) { - float4 __19; - uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4)); - ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]); - ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]); - *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __19; - } - } - #pragma unroll - for (int i_23 = 0; i_23 < 2; ++i_23) { - *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032)); - for (int vec_7 = 0; vec_7 < 2; ++vec_7) { - float4 __20; - float4 __21; - float4 __22; - uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4)); - ((float2*)(&__22))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]); - ((float2*)(&__22))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]); - float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __21.x = (__22.x*v__19.x); - __21.y = (__22.y*v__19.y); - __21.z = (__22.z*v__19.z); - __21.w = (__22.w*v__19.w); - float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4))); - __20.x = (__21.x*v__20.x); - __20.y = (__21.y*v__20.y); - __20.z = (__21.z*v__20.z); - __20.w = (__21.w*v__20.w); - *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __20; - } - } - #pragma unroll - for (int i_24 = 0; i_24 < 2; ++i_24) { - for (int vec_8 = 0; vec_8 < 2; ++vec_8) { - uint2 __23; - float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__23))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__23))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]); - *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __23; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0); - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_25 = 0; i_25 < 2; ++i_25) { - *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104)); - for (int vec_9 = 0; vec_9 < 2; ++vec_9) { - float4 __24; - uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4)); - ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]); - ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]); - *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __24; - } - } - #pragma unroll - for (int i_26 = 0; i_26 < 2; ++i_26) { - *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056)); - for (int vec_10 = 0; vec_10 < 2; ++vec_10) { - float4 __25; - float4 __26; - float4 __27; - uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4)); - ((float2*)(&__27))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]); - ((float2*)(&__27))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]); - float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __26.x = (__27.x*v__24.x); - __26.y = (__27.y*v__24.y); - __26.z = (__27.z*v__24.z); - __26.w = (__27.w*v__24.w); - float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4))); - __25.x = (__26.x*v__25.x); - __25.y = (__26.y*v__25.y); - __25.z = (__26.z*v__25.z); - __25.w = (__26.w*v__25.w); - *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __25; - } - } - #pragma unroll - for (int i_27 = 0; i_27 < 2; ++i_27) { - for (int vec_11 = 0; vec_11 < 2; ++vec_11) { - uint2 __28; - float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__28))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__28))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]); - *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __28; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/0f368e85ad91dc38d1eca212e6350c2e26e5f453a6c404b9ceb10dca001fe742/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/0f368e85ad91dc38d1eca212e6350c2e26e5f453a6c404b9ceb10dca001fe742/host_kernel.cu deleted file mode 100644 index 230868b..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/0f368e85ad91dc38d1eca212e6350c2e26e5f453a6c404b9ceb10dca001fe742/host_kernel.cu +++ /dev/null @@ -1,3359 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_pre_big_fuse_with_norm_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[16]; - void* stack_ffi_any = stack; - if (!((num_args == 9))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_pre_big_fuse_with_norm_tilelang: num_args should be 9", (long long)(num_args), (long long)(9)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang: args pointer is NULL"); - return -1; - } - int32_t gemm_out_mul_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((gemm_out_mul_type_index == 0) || (gemm_out_mul_type_index == 4)) || (gemm_out_mul_type_index == 7)) || (64 <= gemm_out_mul_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_mul expected pointer or tensor handle"); - return -1; - } - int32_t gemm_out_sqrsum_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((gemm_out_sqrsum_type_index == 0) || (gemm_out_sqrsum_type_index == 4)) || (gemm_out_sqrsum_type_index == 7)) || (64 <= gemm_out_sqrsum_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_sqrsum expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t layer_input_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((layer_input_type_index == 0) || (layer_input_type_index == 4)) || (layer_input_type_index == 7)) || (64 <= layer_input_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input layer_input expected pointer or tensor handle"); - return -1; - } - int32_t norm_weight_type_index = (((TVMFFIAny*)args)[8].type_index); - if (!(((((norm_weight_type_index == 0) || (norm_weight_type_index == 4)) || (norm_weight_type_index == 7)) || (64 <= norm_weight_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input norm_weight expected pointer or tensor handle"); - return -1; - } - void* gemm_out_mul = ((gemm_out_mul_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* gemm_out_sqrsum = ((gemm_out_sqrsum_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* layer_input = ((layer_input_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - void* norm_weight = ((norm_weight_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[8].v_ptr) + 24)) : (((TVMFFIAny*)args)[8].v_ptr)); - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null = (gemm_out_mul == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null = (gemm_out_sqrsum == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_residual_is_null = (residual == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null = (layer_input == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.layer_input is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null = (norm_weight == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.norm_weight is expected to have non-NULL pointer"); - return -1; - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape = (((DLTensor*)gemm_out_mul)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape = (((DLTensor*)gemm_out_sqrsum)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape = (((DLTensor*)layer_input)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape = (((DLTensor*)norm_weight)[0].shape); - if ((((DLTensor*)gemm_out_mul)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul, mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum, mhc_pre_big_fuse_with_norm_tilelang.residual, mhc_pre_big_fuse_with_norm_tilelang.post_mix, mhc_pre_big_fuse_with_norm_tilelang.comb_mix, mhc_pre_big_fuse_with_norm_tilelang.layer_input"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides = (((DLTensor*)gemm_out_mul)[0].strides); - int32_t dev_id = (((DLTensor*)gemm_out_mul)[0].device.device_id); - void* gemm_out_mul_1 = (((DLTensor*)gemm_out_mul)[0].data); - if ((((DLTensor*)gemm_out_sqrsum)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides = (((DLTensor*)gemm_out_sqrsum)[0].strides); - void* gemm_out_sqrsum_1 = (((DLTensor*)gemm_out_sqrsum)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)comb_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)layer_input)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides = (((DLTensor*)layer_input)[0].strides); - void* layer_input_1 = (((DLTensor*)layer_input)[0].data); - if ((((DLTensor*)norm_weight)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_9) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides = (((DLTensor*)norm_weight)[0].strides); - void* norm_weight_1 = (((DLTensor*)norm_weight)[0].data); - if ((((((DLTensor*)gemm_out_mul)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_mul)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_mul)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0]) != 64) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)64; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[1]); - } - if ((condval_2 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - int32_t condval_3; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[0]); - } - if (((num_tokens * 24) != condval_3) && (num_tokens != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_15) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_mul)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_mul)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_17) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_18) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)gemm_out_sqrsum)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0]) != 64) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)64; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[1]); - } - if ((condval_4 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_22) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[0]); - } - if ((((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != condval_5) && (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_23) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_sqrsum)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_26) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_27) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_28) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_29) != 0) { - return -1; - } - } - int32_t condval_6; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - if (condval_6 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_7; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_7); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_34) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_35) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_36) != 0) { - return -1; - } - } - int32_t condval_8; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - if (condval_8 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_9; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_9); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_38) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_40) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_41) != 0) { - return -1; - } - } - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_42) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_46) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - if (condval_12 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_13; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_13); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_47) != 0) { - return -1; - } - } - int32_t condval_14; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[0]); - } - if ((condval_14 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_48) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_49) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_51) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_52) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_53) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_54) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - if (condval_15 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_16; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_16); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_56) != 0) { - return -1; - } - } - int32_t condval_17; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[0]); - } - if ((condval_17 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_58) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_60) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_61) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_62) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_63) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1]) != 16) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_64) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - if (condval_18 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[0]); - } - if ((condval_20 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_66) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_67) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_68) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_69) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_70) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)layer_input)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)layer_input)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)layer_input)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_71) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_72) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_73) != 0) { - return -1; - } - } - int32_t condval_21; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - if (condval_21 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_22; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_22); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - int32_t condval_23; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[0]); - } - if ((condval_23 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_75) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)layer_input)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_76) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_77) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_78) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_79) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)norm_weight)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)norm_weight)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)norm_weight)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_80) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - if (condval_24 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_25; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_25); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)norm_weight)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_83) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_85) != 0) { - return -1; - } - } - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_86) != 0) { - return -1; - } - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_87) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = gemm_out_mul_1; - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = gemm_out_sqrsum_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = hc_scale_1; - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = layer_input_1; - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = norm_weight_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = post_mix_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)96); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)29040); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = (int64_t)0; - if (mhc_pre_big_fuse_with_norm_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_pre_big_fuse_with_norm_tilelang_kernel", &mhc_pre_big_fuse_with_norm_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_pre_big_fuse_with_norm_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 15, &result_88) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/0f368e85ad91dc38d1eca212e6350c2e26e5f453a6c404b9ceb10dca001fe742/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/0f368e85ad91dc38d1eca212e6350c2e26e5f453a6c404b9ceb10dca001fe742/params.pkl deleted file mode 100644 index 4e19deaa77accf55e9131f095df4a3ef92ed074e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2200 zcmeHI&r2IY6xPP1fmjKYQY{E}PAP=NAJ9Wi9=r_|X~m0#VK)1cnYuf(%uZS*Ea<_W z##5$$U;mZ9-6U#NsN@>=HjnSU?|a|dH*;D2eN$g5zpih&cUHy_b2SJ+4Wxn~<%TET zH7FPb@lkmi`%hOsM|3r^7b$qRW`88C-OJudKrQMkmY;(-{01$sVBV2i)@Gf`07jE4oFOt^ew?Up{ozOmK{2 zKXhNLxdv14lj-m~^elqQLV=Ygfb-ONS2^JZ!`GONB5Q)H2pQd<5iwd@CRGIIq_fv1 zN=J~9&gqC0U6T$u)Cx%6zC9NXe6vQX0cn#6 -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float mixes[1]; - float rms[1]; - float cm[1]; - float row_max[1]; - float row_sum[1]; - float col_sum[1]; - float sumsq_per_pos[16]; - float sumsq[1]; - float rsqrt_norm[1]; - float xl[64]; - float ol[16]; - bfloat16_t xs_local_cast[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t xs_local_cast_2[8]; - float w_local[16]; - float ol_1[16]; - bfloat16_t w_shared_local_cast_3[8]; - bfloat16_t output_shared_local_cast_4[8]; - bfloat16_t layer_input_local_cast_5[8]; - bfloat16_t w_shared_local_cast_6[8]; - bfloat16_t output_shared_local_cast_7[8]; - bfloat16_t layer_input_local_cast_8[8]; - bfloat16_t w_shared_local_cast_9[8]; - bfloat16_t output_shared_local_cast_10[8]; - bfloat16_t layer_input_local_cast_11[8]; - mixes[0] = 0x0p+0f/*0.000000e+00*/; - rms[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - for (int i_split = 0; i_split < 11; ++i_split) { - rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]); - } - rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - mixes[0] = 0x0p+0f/*0.000000e+00*/; - for (int i_split_1 = 0; i_split_1 < 11; ++i_split_1) { - mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]); - } - mixes[0] = (mixes[0] * rms[0]); - if ((((int)threadIdx.x) / 24) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0]; - } - __syncthreads(); - if (((int)threadIdx.x) < 32) { - if (((int)threadIdx.x) < 4) { - post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/); - } - cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]); - row_max[0] = -CUDART_INF_F; - row_max[0] = max(row_max[0], cm[0]); - row_max[0] = tl::AllReduce>::run(row_max[0]); - cm[0] = expf((cm[0] - row_max[0])); - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - for (int __1 = 0; __1 < 19; ++__1) { - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - } - if ((((int)threadIdx.x) >> 4) == 0) { - comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0]; - } - } else { - if (((int)threadIdx.x) < 36) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - #pragma unroll - for (int i = 0; i < 4; ++i) { - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_2 = 0; i_2 < 8; ++i_2) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_4 = 0; i_4 < 8; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - tl::__sync_thread_partial<3, 64>(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)]; - #pragma unroll - for (int i_6 = 0; i_6 < 16; ++i_6) { - ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)])); - } - } - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - float4 __3; - float4 v__1 = *(float4*)(sumsq_per_pos + (i_7 * 4)); - float4 __4; - float4 v__2 = *(float4*)(ol + (i_7 * 4)); - __4.x = (v__2.x*v__2.x); - __4.y = (v__2.y*v__2.y); - __4.z = (v__2.z*v__2.z); - __4.w = (v__2.w*v__2.w); - __3.x = (v__1.x+__4.x); - __3.y = (v__1.y+__4.y); - __3.z = (v__1.z+__4.z); - __3.w = (v__1.w+__4.w); - *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3; - uint2 __5; - float4 v__3 = *(float4*)(ol + (i_7 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __5; - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_8 = 0; i_8 < 8; ++i_8) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __6; - uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]); - *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __6; - } - } - #pragma unroll - for (int i_9 = 0; i_9 < 4; ++i_9) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)]; - #pragma unroll - for (int i_10 = 0; i_10 < 16; ++i_10) { - ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)])); - } - } - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - float4 __7; - float4 v__5 = *(float4*)(sumsq_per_pos + (i_11 * 4)); - float4 __8; - float4 v__6 = *(float4*)(ol + (i_11 * 4)); - __8.x = (v__6.x*v__6.x); - __8.y = (v__6.y*v__6.y); - __8.z = (v__6.z*v__6.z); - __8.w = (v__6.w*v__6.w); - __7.x = (v__5.x+__8.x); - __7.y = (v__5.y+__8.y); - __7.z = (v__5.z+__8.z); - __7.w = (v__5.w+__8.w); - *(float4*)(sumsq_per_pos + (i_11 * 4)) = __7; - uint2 __9; - float4 v__7 = *(float4*)(ol + (i_11 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__9))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__9))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __9; - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_12 = 0; i_12 < 8; ++i_12) { - *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __10; - uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4)); - ((float2*)(&__10))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]); - ((float2*)(&__10))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]); - *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __10; - } - } - #pragma unroll - for (int i_13 = 0; i_13 < 4; ++i_13) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)]; - #pragma unroll - for (int i_14 = 0; i_14 < 16; ++i_14) { - ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)])); - } - } - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - float4 __11; - float4 v__9 = *(float4*)(sumsq_per_pos + (i_15 * 4)); - float4 __12; - float4 v__10 = *(float4*)(ol + (i_15 * 4)); - __12.x = (v__10.x*v__10.x); - __12.y = (v__10.y*v__10.y); - __12.z = (v__10.z*v__10.z); - __12.w = (v__10.w*v__10.w); - __11.x = (v__9.x+__12.x); - __11.y = (v__9.y+__12.y); - __11.z = (v__9.z+__12.z); - __11.w = (v__9.w+__12.w); - *(float4*)(sumsq_per_pos + (i_15 * 4)) = __11; - uint2 __13; - float4 v__11 = *(float4*)(ol + (i_15 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__13))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__13))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __13; - } - sumsq[0] = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int rv = 0; rv < 16; ++rv) { - sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]); - } - tl::__sync_thread_partial<3, 64>(); - sumsq[0] = tl::AllReduce>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192]))); - rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_17 = 0; i_17 < 2; ++i_17) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)]))); - } - tl::cp_async_commit(); - for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_18 = 0; i_18 < 2; ++i_18) { - *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_3 = 0; vec_3 < 2; ++vec_3) { - float4 __14; - uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4)); - ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]); - ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]); - *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __14; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_19 = 0; i_19 < 2; ++i_19) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)]))); - } - tl::cp_async_commit(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_20 = 0; i_20 < 2; ++i_20) { - *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984)); - for (int vec_4 = 0; vec_4 < 2; ++vec_4) { - float4 __15; - float4 __16; - float4 __17; - uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4)); - ((float2*)(&__17))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]); - ((float2*)(&__17))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]); - float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __16.x = (__17.x*v__14.x); - __16.y = (__17.y*v__14.y); - __16.z = (__17.z*v__14.z); - __16.w = (__17.w*v__14.w); - float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4))); - __15.x = (__16.x*v__15.x); - __15.y = (__16.y*v__15.y); - __15.z = (__16.z*v__15.z); - __15.w = (__16.w*v__15.w); - *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __15; - } - } - #pragma unroll - for (int i_21 = 0; i_21 < 2; ++i_21) { - for (int vec_5 = 0; vec_5 < 2; ++vec_5) { - uint2 __18; - float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__18))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__18))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]); - *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __18; - } - *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0); - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_22 = 0; i_22 < 2; ++i_22) { - *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_6 = 0; vec_6 < 2; ++vec_6) { - float4 __19; - uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4)); - ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]); - ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]); - *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __19; - } - } - #pragma unroll - for (int i_23 = 0; i_23 < 2; ++i_23) { - *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032)); - for (int vec_7 = 0; vec_7 < 2; ++vec_7) { - float4 __20; - float4 __21; - float4 __22; - uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4)); - ((float2*)(&__22))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]); - ((float2*)(&__22))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]); - float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __21.x = (__22.x*v__19.x); - __21.y = (__22.y*v__19.y); - __21.z = (__22.z*v__19.z); - __21.w = (__22.w*v__19.w); - float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4))); - __20.x = (__21.x*v__20.x); - __20.y = (__21.y*v__20.y); - __20.z = (__21.z*v__20.z); - __20.w = (__21.w*v__20.w); - *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __20; - } - } - #pragma unroll - for (int i_24 = 0; i_24 < 2; ++i_24) { - for (int vec_8 = 0; vec_8 < 2; ++vec_8) { - uint2 __23; - float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__23))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__23))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]); - *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __23; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0); - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_25 = 0; i_25 < 2; ++i_25) { - *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104)); - for (int vec_9 = 0; vec_9 < 2; ++vec_9) { - float4 __24; - uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4)); - ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]); - ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]); - *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __24; - } - } - #pragma unroll - for (int i_26 = 0; i_26 < 2; ++i_26) { - *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056)); - for (int vec_10 = 0; vec_10 < 2; ++vec_10) { - float4 __25; - float4 __26; - float4 __27; - uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4)); - ((float2*)(&__27))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]); - ((float2*)(&__27))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]); - float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __26.x = (__27.x*v__24.x); - __26.y = (__27.y*v__24.y); - __26.z = (__27.z*v__24.z); - __26.w = (__27.w*v__24.w); - float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4))); - __25.x = (__26.x*v__25.x); - __25.y = (__26.y*v__25.y); - __25.z = (__26.z*v__25.z); - __25.w = (__26.w*v__25.w); - *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __25; - } - } - #pragma unroll - for (int i_27 = 0; i_27 < 2; ++i_27) { - for (int vec_11 = 0; vec_11 < 2; ++vec_11) { - uint2 __28; - float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__28))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__28))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]); - *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __28; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/147eb04270024c36acaf2922c18e38bbdfc222673231f206ebd108766e937d6c/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/147eb04270024c36acaf2922c18e38bbdfc222673231f206ebd108766e937d6c/host_kernel.cu deleted file mode 100644 index 2c40496..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/147eb04270024c36acaf2922c18e38bbdfc222673231f206ebd108766e937d6c/host_kernel.cu +++ /dev/null @@ -1,3359 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_pre_big_fuse_with_norm_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[16]; - void* stack_ffi_any = stack; - if (!((num_args == 9))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_pre_big_fuse_with_norm_tilelang: num_args should be 9", (long long)(num_args), (long long)(9)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang: args pointer is NULL"); - return -1; - } - int32_t gemm_out_mul_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((gemm_out_mul_type_index == 0) || (gemm_out_mul_type_index == 4)) || (gemm_out_mul_type_index == 7)) || (64 <= gemm_out_mul_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_mul expected pointer or tensor handle"); - return -1; - } - int32_t gemm_out_sqrsum_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((gemm_out_sqrsum_type_index == 0) || (gemm_out_sqrsum_type_index == 4)) || (gemm_out_sqrsum_type_index == 7)) || (64 <= gemm_out_sqrsum_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_sqrsum expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t layer_input_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((layer_input_type_index == 0) || (layer_input_type_index == 4)) || (layer_input_type_index == 7)) || (64 <= layer_input_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input layer_input expected pointer or tensor handle"); - return -1; - } - int32_t norm_weight_type_index = (((TVMFFIAny*)args)[8].type_index); - if (!(((((norm_weight_type_index == 0) || (norm_weight_type_index == 4)) || (norm_weight_type_index == 7)) || (64 <= norm_weight_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input norm_weight expected pointer or tensor handle"); - return -1; - } - void* gemm_out_mul = ((gemm_out_mul_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* gemm_out_sqrsum = ((gemm_out_sqrsum_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* layer_input = ((layer_input_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - void* norm_weight = ((norm_weight_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[8].v_ptr) + 24)) : (((TVMFFIAny*)args)[8].v_ptr)); - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null = (gemm_out_mul == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null = (gemm_out_sqrsum == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_residual_is_null = (residual == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null = (layer_input == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.layer_input is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null = (norm_weight == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.norm_weight is expected to have non-NULL pointer"); - return -1; - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape = (((DLTensor*)gemm_out_mul)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape = (((DLTensor*)gemm_out_sqrsum)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape = (((DLTensor*)layer_input)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape = (((DLTensor*)norm_weight)[0].shape); - if ((((DLTensor*)gemm_out_mul)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul, mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum, mhc_pre_big_fuse_with_norm_tilelang.residual, mhc_pre_big_fuse_with_norm_tilelang.post_mix, mhc_pre_big_fuse_with_norm_tilelang.comb_mix, mhc_pre_big_fuse_with_norm_tilelang.layer_input"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides = (((DLTensor*)gemm_out_mul)[0].strides); - int32_t dev_id = (((DLTensor*)gemm_out_mul)[0].device.device_id); - void* gemm_out_mul_1 = (((DLTensor*)gemm_out_mul)[0].data); - if ((((DLTensor*)gemm_out_sqrsum)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides = (((DLTensor*)gemm_out_sqrsum)[0].strides); - void* gemm_out_sqrsum_1 = (((DLTensor*)gemm_out_sqrsum)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)comb_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)layer_input)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides = (((DLTensor*)layer_input)[0].strides); - void* layer_input_1 = (((DLTensor*)layer_input)[0].data); - if ((((DLTensor*)norm_weight)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_9) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides = (((DLTensor*)norm_weight)[0].strides); - void* norm_weight_1 = (((DLTensor*)norm_weight)[0].data); - if ((((((DLTensor*)gemm_out_mul)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_mul)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_mul)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0]) != 11) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)11; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[1]); - } - if ((condval_2 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - int32_t condval_3; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[0]); - } - if (((num_tokens * 24) != condval_3) && (num_tokens != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_15) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_mul)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_mul)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_17) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_18) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)gemm_out_sqrsum)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0]) != 11) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)11; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[1]); - } - if ((condval_4 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_22) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[0]); - } - if ((((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != condval_5) && (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_23) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_sqrsum)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_26) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_27) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_28) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_29) != 0) { - return -1; - } - } - int32_t condval_6; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - if (condval_6 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_7; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_7); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_34) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_35) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_36) != 0) { - return -1; - } - } - int32_t condval_8; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - if (condval_8 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_9; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_9); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_38) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_40) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_41) != 0) { - return -1; - } - } - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_42) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_46) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - if (condval_12 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_13; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_13); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_47) != 0) { - return -1; - } - } - int32_t condval_14; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[0]); - } - if ((condval_14 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_48) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_49) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_51) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_52) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_53) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_54) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - if (condval_15 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_16; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_16); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_56) != 0) { - return -1; - } - } - int32_t condval_17; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[0]); - } - if ((condval_17 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_58) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_60) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_61) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_62) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_63) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1]) != 16) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_64) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - if (condval_18 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[0]); - } - if ((condval_20 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_66) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_67) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_68) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_69) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_70) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)layer_input)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)layer_input)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)layer_input)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_71) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_72) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_73) != 0) { - return -1; - } - } - int32_t condval_21; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - if (condval_21 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_22; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_22); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - int32_t condval_23; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[0]); - } - if ((condval_23 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_75) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)layer_input)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_76) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_77) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_78) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_79) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)norm_weight)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)norm_weight)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)norm_weight)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_80) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - if (condval_24 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_25; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_25); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)norm_weight)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_83) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_85) != 0) { - return -1; - } - } - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_86) != 0) { - return -1; - } - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_87) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = gemm_out_mul_1; - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = gemm_out_sqrsum_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = hc_scale_1; - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = layer_input_1; - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = norm_weight_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = post_mix_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)96); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)29040); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = (int64_t)0; - if (mhc_pre_big_fuse_with_norm_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_pre_big_fuse_with_norm_tilelang_kernel", &mhc_pre_big_fuse_with_norm_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_pre_big_fuse_with_norm_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 15, &result_88) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/147eb04270024c36acaf2922c18e38bbdfc222673231f206ebd108766e937d6c/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/147eb04270024c36acaf2922c18e38bbdfc222673231f206ebd108766e937d6c/params.pkl deleted file mode 100644 index 4a4539c7c04f3e17906ce599038dafa4ec90d55a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2200 zcmeHIO=}x55KZIQHMk9i5Rw)OtxgV@C5a&ipK}upp$)l|Ahh1GrDRuHv~rqa7MeqP zQcqTYU;j#H*B^D0K!dN&-ZZ?K_vX!x^u7A)XML^wE_};vTNy*l)gS;hkP3p78=iRA zpkNflr{!tv_ilZT=xSuYrQqGBy-rxaF9S9Sh}zh@m;G3C+d1@i{??!5h+P`mv|$^gVrH{U@LO@S&?_f@2)} zp$)O=8ce}gro&6vTia?TBg?=Txh)&y4(GP>U*VzjnQst9gKx6>v{ zN05>3mk}wtCS7u(6_C6=KNk*swnnM}X_E-$Aw~_c2Ym%9TSOxHsgX$`e-0gyEai%H z_c3=$AQk#Y@&$@QHtPL<63XQXiacnSY8h8bTW+OBhQmC+$bA;lT^NR -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float mixes[1]; - float rms[1]; - float cm[1]; - float row_max[1]; - float row_sum[1]; - float col_sum[1]; - float sumsq_per_pos[16]; - float sumsq[1]; - float rsqrt_norm[1]; - float xl[64]; - float ol[16]; - bfloat16_t xs_local_cast[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t xs_local_cast_2[8]; - float w_local[16]; - float ol_1[16]; - bfloat16_t w_shared_local_cast_3[8]; - bfloat16_t output_shared_local_cast_4[8]; - bfloat16_t layer_input_local_cast_5[8]; - bfloat16_t w_shared_local_cast_6[8]; - bfloat16_t output_shared_local_cast_7[8]; - bfloat16_t layer_input_local_cast_8[8]; - bfloat16_t w_shared_local_cast_9[8]; - bfloat16_t output_shared_local_cast_10[8]; - bfloat16_t layer_input_local_cast_11[8]; - mixes[0] = 0x0p+0f/*0.000000e+00*/; - rms[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - for (int i_split = 0; i_split < 19; ++i_split) { - rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]); - } - rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - mixes[0] = 0x0p+0f/*0.000000e+00*/; - for (int i_split_1 = 0; i_split_1 < 19; ++i_split_1) { - mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]); - } - mixes[0] = (mixes[0] * rms[0]); - if ((((int)threadIdx.x) / 24) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0]; - } - __syncthreads(); - if (((int)threadIdx.x) < 32) { - if (((int)threadIdx.x) < 4) { - post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/); - } - cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]); - row_max[0] = -CUDART_INF_F; - row_max[0] = max(row_max[0], cm[0]); - row_max[0] = tl::AllReduce>::run(row_max[0]); - cm[0] = expf((cm[0] - row_max[0])); - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - for (int __1 = 0; __1 < 19; ++__1) { - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - } - if ((((int)threadIdx.x) >> 4) == 0) { - comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0]; - } - } else { - if (((int)threadIdx.x) < 36) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - #pragma unroll - for (int i = 0; i < 4; ++i) { - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_2 = 0; i_2 < 8; ++i_2) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_4 = 0; i_4 < 8; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - tl::__sync_thread_partial<3, 64>(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)]; - #pragma unroll - for (int i_6 = 0; i_6 < 16; ++i_6) { - ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)])); - } - } - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - float4 __3; - float4 v__1 = *(float4*)(sumsq_per_pos + (i_7 * 4)); - float4 __4; - float4 v__2 = *(float4*)(ol + (i_7 * 4)); - __4.x = (v__2.x*v__2.x); - __4.y = (v__2.y*v__2.y); - __4.z = (v__2.z*v__2.z); - __4.w = (v__2.w*v__2.w); - __3.x = (v__1.x+__4.x); - __3.y = (v__1.y+__4.y); - __3.z = (v__1.z+__4.z); - __3.w = (v__1.w+__4.w); - *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3; - uint2 __5; - float4 v__3 = *(float4*)(ol + (i_7 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __5; - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_8 = 0; i_8 < 8; ++i_8) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __6; - uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]); - *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __6; - } - } - #pragma unroll - for (int i_9 = 0; i_9 < 4; ++i_9) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)]; - #pragma unroll - for (int i_10 = 0; i_10 < 16; ++i_10) { - ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)])); - } - } - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - float4 __7; - float4 v__5 = *(float4*)(sumsq_per_pos + (i_11 * 4)); - float4 __8; - float4 v__6 = *(float4*)(ol + (i_11 * 4)); - __8.x = (v__6.x*v__6.x); - __8.y = (v__6.y*v__6.y); - __8.z = (v__6.z*v__6.z); - __8.w = (v__6.w*v__6.w); - __7.x = (v__5.x+__8.x); - __7.y = (v__5.y+__8.y); - __7.z = (v__5.z+__8.z); - __7.w = (v__5.w+__8.w); - *(float4*)(sumsq_per_pos + (i_11 * 4)) = __7; - uint2 __9; - float4 v__7 = *(float4*)(ol + (i_11 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__9))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__9))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __9; - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_12 = 0; i_12 < 8; ++i_12) { - *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __10; - uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4)); - ((float2*)(&__10))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]); - ((float2*)(&__10))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]); - *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __10; - } - } - #pragma unroll - for (int i_13 = 0; i_13 < 4; ++i_13) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)]; - #pragma unroll - for (int i_14 = 0; i_14 < 16; ++i_14) { - ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)])); - } - } - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - float4 __11; - float4 v__9 = *(float4*)(sumsq_per_pos + (i_15 * 4)); - float4 __12; - float4 v__10 = *(float4*)(ol + (i_15 * 4)); - __12.x = (v__10.x*v__10.x); - __12.y = (v__10.y*v__10.y); - __12.z = (v__10.z*v__10.z); - __12.w = (v__10.w*v__10.w); - __11.x = (v__9.x+__12.x); - __11.y = (v__9.y+__12.y); - __11.z = (v__9.z+__12.z); - __11.w = (v__9.w+__12.w); - *(float4*)(sumsq_per_pos + (i_15 * 4)) = __11; - uint2 __13; - float4 v__11 = *(float4*)(ol + (i_15 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__13))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__13))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __13; - } - sumsq[0] = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int rv = 0; rv < 16; ++rv) { - sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]); - } - tl::__sync_thread_partial<3, 64>(); - sumsq[0] = tl::AllReduce>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192]))); - rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_17 = 0; i_17 < 2; ++i_17) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)]))); - } - tl::cp_async_commit(); - for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_18 = 0; i_18 < 2; ++i_18) { - *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_3 = 0; vec_3 < 2; ++vec_3) { - float4 __14; - uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4)); - ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]); - ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]); - *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __14; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_19 = 0; i_19 < 2; ++i_19) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)]))); - } - tl::cp_async_commit(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_20 = 0; i_20 < 2; ++i_20) { - *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984)); - for (int vec_4 = 0; vec_4 < 2; ++vec_4) { - float4 __15; - float4 __16; - float4 __17; - uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4)); - ((float2*)(&__17))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]); - ((float2*)(&__17))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]); - float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __16.x = (__17.x*v__14.x); - __16.y = (__17.y*v__14.y); - __16.z = (__17.z*v__14.z); - __16.w = (__17.w*v__14.w); - float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4))); - __15.x = (__16.x*v__15.x); - __15.y = (__16.y*v__15.y); - __15.z = (__16.z*v__15.z); - __15.w = (__16.w*v__15.w); - *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __15; - } - } - #pragma unroll - for (int i_21 = 0; i_21 < 2; ++i_21) { - for (int vec_5 = 0; vec_5 < 2; ++vec_5) { - uint2 __18; - float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__18))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__18))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]); - *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __18; - } - *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0); - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_22 = 0; i_22 < 2; ++i_22) { - *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_6 = 0; vec_6 < 2; ++vec_6) { - float4 __19; - uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4)); - ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]); - ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]); - *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __19; - } - } - #pragma unroll - for (int i_23 = 0; i_23 < 2; ++i_23) { - *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032)); - for (int vec_7 = 0; vec_7 < 2; ++vec_7) { - float4 __20; - float4 __21; - float4 __22; - uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4)); - ((float2*)(&__22))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]); - ((float2*)(&__22))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]); - float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __21.x = (__22.x*v__19.x); - __21.y = (__22.y*v__19.y); - __21.z = (__22.z*v__19.z); - __21.w = (__22.w*v__19.w); - float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4))); - __20.x = (__21.x*v__20.x); - __20.y = (__21.y*v__20.y); - __20.z = (__21.z*v__20.z); - __20.w = (__21.w*v__20.w); - *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __20; - } - } - #pragma unroll - for (int i_24 = 0; i_24 < 2; ++i_24) { - for (int vec_8 = 0; vec_8 < 2; ++vec_8) { - uint2 __23; - float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__23))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__23))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]); - *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __23; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0); - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_25 = 0; i_25 < 2; ++i_25) { - *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104)); - for (int vec_9 = 0; vec_9 < 2; ++vec_9) { - float4 __24; - uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4)); - ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]); - ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]); - *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __24; - } - } - #pragma unroll - for (int i_26 = 0; i_26 < 2; ++i_26) { - *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056)); - for (int vec_10 = 0; vec_10 < 2; ++vec_10) { - float4 __25; - float4 __26; - float4 __27; - uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4)); - ((float2*)(&__27))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]); - ((float2*)(&__27))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]); - float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __26.x = (__27.x*v__24.x); - __26.y = (__27.y*v__24.y); - __26.z = (__27.z*v__24.z); - __26.w = (__27.w*v__24.w); - float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4))); - __25.x = (__26.x*v__25.x); - __25.y = (__26.y*v__25.y); - __25.z = (__26.z*v__25.z); - __25.w = (__26.w*v__25.w); - *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __25; - } - } - #pragma unroll - for (int i_27 = 0; i_27 < 2; ++i_27) { - for (int vec_11 = 0; vec_11 < 2; ++vec_11) { - uint2 __28; - float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__28))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__28))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]); - *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __28; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1eae71a46e812d68441e1ebddd96e5dbecfaeec6b4f8ed7749c4f21be97720e8/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1eae71a46e812d68441e1ebddd96e5dbecfaeec6b4f8ed7749c4f21be97720e8/host_kernel.cu deleted file mode 100644 index 1f028bd..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1eae71a46e812d68441e1ebddd96e5dbecfaeec6b4f8ed7749c4f21be97720e8/host_kernel.cu +++ /dev/null @@ -1,3359 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_pre_big_fuse_with_norm_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[16]; - void* stack_ffi_any = stack; - if (!((num_args == 9))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_pre_big_fuse_with_norm_tilelang: num_args should be 9", (long long)(num_args), (long long)(9)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang: args pointer is NULL"); - return -1; - } - int32_t gemm_out_mul_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((gemm_out_mul_type_index == 0) || (gemm_out_mul_type_index == 4)) || (gemm_out_mul_type_index == 7)) || (64 <= gemm_out_mul_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_mul expected pointer or tensor handle"); - return -1; - } - int32_t gemm_out_sqrsum_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((gemm_out_sqrsum_type_index == 0) || (gemm_out_sqrsum_type_index == 4)) || (gemm_out_sqrsum_type_index == 7)) || (64 <= gemm_out_sqrsum_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_sqrsum expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t layer_input_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((layer_input_type_index == 0) || (layer_input_type_index == 4)) || (layer_input_type_index == 7)) || (64 <= layer_input_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input layer_input expected pointer or tensor handle"); - return -1; - } - int32_t norm_weight_type_index = (((TVMFFIAny*)args)[8].type_index); - if (!(((((norm_weight_type_index == 0) || (norm_weight_type_index == 4)) || (norm_weight_type_index == 7)) || (64 <= norm_weight_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input norm_weight expected pointer or tensor handle"); - return -1; - } - void* gemm_out_mul = ((gemm_out_mul_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* gemm_out_sqrsum = ((gemm_out_sqrsum_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* layer_input = ((layer_input_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - void* norm_weight = ((norm_weight_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[8].v_ptr) + 24)) : (((TVMFFIAny*)args)[8].v_ptr)); - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null = (gemm_out_mul == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null = (gemm_out_sqrsum == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_residual_is_null = (residual == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null = (layer_input == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.layer_input is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null = (norm_weight == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.norm_weight is expected to have non-NULL pointer"); - return -1; - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape = (((DLTensor*)gemm_out_mul)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape = (((DLTensor*)gemm_out_sqrsum)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape = (((DLTensor*)layer_input)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape = (((DLTensor*)norm_weight)[0].shape); - if ((((DLTensor*)gemm_out_mul)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul, mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum, mhc_pre_big_fuse_with_norm_tilelang.residual, mhc_pre_big_fuse_with_norm_tilelang.post_mix, mhc_pre_big_fuse_with_norm_tilelang.comb_mix, mhc_pre_big_fuse_with_norm_tilelang.layer_input"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides = (((DLTensor*)gemm_out_mul)[0].strides); - int32_t dev_id = (((DLTensor*)gemm_out_mul)[0].device.device_id); - void* gemm_out_mul_1 = (((DLTensor*)gemm_out_mul)[0].data); - if ((((DLTensor*)gemm_out_sqrsum)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides = (((DLTensor*)gemm_out_sqrsum)[0].strides); - void* gemm_out_sqrsum_1 = (((DLTensor*)gemm_out_sqrsum)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)comb_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)layer_input)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides = (((DLTensor*)layer_input)[0].strides); - void* layer_input_1 = (((DLTensor*)layer_input)[0].data); - if ((((DLTensor*)norm_weight)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_9) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides = (((DLTensor*)norm_weight)[0].strides); - void* norm_weight_1 = (((DLTensor*)norm_weight)[0].data); - if ((((((DLTensor*)gemm_out_mul)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_mul)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_mul)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0]) != 19) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)19; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[1]); - } - if ((condval_2 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - int32_t condval_3; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[0]); - } - if (((num_tokens * 24) != condval_3) && (num_tokens != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_15) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_mul)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_mul)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_17) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_18) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)gemm_out_sqrsum)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0]) != 19) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)19; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[1]); - } - if ((condval_4 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_22) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[0]); - } - if ((((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != condval_5) && (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_23) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_sqrsum)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_26) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_27) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_28) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_29) != 0) { - return -1; - } - } - int32_t condval_6; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - if (condval_6 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_7; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_7); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_34) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_35) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_36) != 0) { - return -1; - } - } - int32_t condval_8; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - if (condval_8 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_9; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_9); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_38) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_40) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_41) != 0) { - return -1; - } - } - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_42) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_46) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - if (condval_12 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_13; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_13); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_47) != 0) { - return -1; - } - } - int32_t condval_14; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[0]); - } - if ((condval_14 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_48) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_49) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_51) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_52) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_53) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_54) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - if (condval_15 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_16; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_16); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_56) != 0) { - return -1; - } - } - int32_t condval_17; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[0]); - } - if ((condval_17 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_58) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_60) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_61) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_62) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_63) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1]) != 16) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_64) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - if (condval_18 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[0]); - } - if ((condval_20 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_66) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_67) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_68) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_69) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_70) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)layer_input)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)layer_input)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)layer_input)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_71) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_72) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_73) != 0) { - return -1; - } - } - int32_t condval_21; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - if (condval_21 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_22; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_22); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - int32_t condval_23; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[0]); - } - if ((condval_23 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_75) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)layer_input)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_76) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_77) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_78) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_79) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)norm_weight)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)norm_weight)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)norm_weight)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_80) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - if (condval_24 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_25; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_25); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)norm_weight)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_83) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_85) != 0) { - return -1; - } - } - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_86) != 0) { - return -1; - } - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_87) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = gemm_out_mul_1; - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = gemm_out_sqrsum_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = hc_scale_1; - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = layer_input_1; - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = norm_weight_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = post_mix_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)96); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)29040); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = (int64_t)0; - if (mhc_pre_big_fuse_with_norm_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_pre_big_fuse_with_norm_tilelang_kernel", &mhc_pre_big_fuse_with_norm_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_pre_big_fuse_with_norm_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 15, &result_88) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1eae71a46e812d68441e1ebddd96e5dbecfaeec6b4f8ed7749c4f21be97720e8/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1eae71a46e812d68441e1ebddd96e5dbecfaeec6b4f8ed7749c4f21be97720e8/params.pkl deleted file mode 100644 index 65b203c4f9564debb91f54c05ba96b6b68e812c8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2200 zcmeHIO-my|5Op^u2K*o(;s*$8pCW`|RoFw$-kKG0#fwPOlc}V8ccy#j>8wf^SP$;0 z_muYc`B%1PlBlb~O0IEkUA(II>eW>Duj<{O`ds;4_@-O4GKQF|UI3~m6$B|aJn?Rk zf>973m#3lMz41AstC77|jYD4c9y0PZ=VBg>R8-J7|ij84olv|pl zb+o}Mo2+AiTVS2C3hNr3FxyXK@IxefhOQrS%TEjXp{F0|C;Gwq6QBL|p{r(sV;uXT zt7641G6k1Rhu@)N5nL7utTX{!rN+C;88;Zd#cUK=6I?~e=L5nCI zK}Oo=15$KN+T=(pAbIQITsZK}8mW4uMIw}k7}duf^bb(kQzVif8<`aHAD{!0rCgEr zXUrWFNQM59{0T)N8} -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float mixes[1]; - float rms[1]; - float cm[1]; - float row_max[1]; - float row_sum[1]; - float col_sum[1]; - float sumsq_per_pos[16]; - float sumsq[1]; - float rsqrt_norm[1]; - float xl[64]; - float ol[16]; - bfloat16_t xs_local_cast[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t xs_local_cast_2[8]; - float w_local[16]; - float ol_1[16]; - bfloat16_t w_shared_local_cast_3[8]; - bfloat16_t output_shared_local_cast_4[8]; - bfloat16_t layer_input_local_cast_5[8]; - bfloat16_t w_shared_local_cast_6[8]; - bfloat16_t output_shared_local_cast_7[8]; - bfloat16_t layer_input_local_cast_8[8]; - bfloat16_t w_shared_local_cast_9[8]; - bfloat16_t output_shared_local_cast_10[8]; - bfloat16_t layer_input_local_cast_11[8]; - mixes[0] = 0x0p+0f/*0.000000e+00*/; - rms[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - for (int i_split = 0; i_split < 6; ++i_split) { - rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]); - } - rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - mixes[0] = 0x0p+0f/*0.000000e+00*/; - for (int i_split_1 = 0; i_split_1 < 6; ++i_split_1) { - mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]); - } - mixes[0] = (mixes[0] * rms[0]); - if ((((int)threadIdx.x) / 24) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0]; - } - __syncthreads(); - if (((int)threadIdx.x) < 32) { - if (((int)threadIdx.x) < 4) { - post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/); - } - cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]); - row_max[0] = -CUDART_INF_F; - row_max[0] = max(row_max[0], cm[0]); - row_max[0] = tl::AllReduce>::run(row_max[0]); - cm[0] = expf((cm[0] - row_max[0])); - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - for (int __1 = 0; __1 < 19; ++__1) { - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - } - if ((((int)threadIdx.x) >> 4) == 0) { - comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0]; - } - } else { - if (((int)threadIdx.x) < 36) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - #pragma unroll - for (int i = 0; i < 4; ++i) { - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_2 = 0; i_2 < 8; ++i_2) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_4 = 0; i_4 < 8; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - tl::__sync_thread_partial<3, 64>(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)]; - #pragma unroll - for (int i_6 = 0; i_6 < 16; ++i_6) { - ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)])); - } - } - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - float4 __3; - float4 v__1 = *(float4*)(sumsq_per_pos + (i_7 * 4)); - float4 __4; - float4 v__2 = *(float4*)(ol + (i_7 * 4)); - __4.x = (v__2.x*v__2.x); - __4.y = (v__2.y*v__2.y); - __4.z = (v__2.z*v__2.z); - __4.w = (v__2.w*v__2.w); - __3.x = (v__1.x+__4.x); - __3.y = (v__1.y+__4.y); - __3.z = (v__1.z+__4.z); - __3.w = (v__1.w+__4.w); - *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3; - uint2 __5; - float4 v__3 = *(float4*)(ol + (i_7 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __5; - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_8 = 0; i_8 < 8; ++i_8) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __6; - uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]); - *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __6; - } - } - #pragma unroll - for (int i_9 = 0; i_9 < 4; ++i_9) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)]; - #pragma unroll - for (int i_10 = 0; i_10 < 16; ++i_10) { - ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)])); - } - } - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - float4 __7; - float4 v__5 = *(float4*)(sumsq_per_pos + (i_11 * 4)); - float4 __8; - float4 v__6 = *(float4*)(ol + (i_11 * 4)); - __8.x = (v__6.x*v__6.x); - __8.y = (v__6.y*v__6.y); - __8.z = (v__6.z*v__6.z); - __8.w = (v__6.w*v__6.w); - __7.x = (v__5.x+__8.x); - __7.y = (v__5.y+__8.y); - __7.z = (v__5.z+__8.z); - __7.w = (v__5.w+__8.w); - *(float4*)(sumsq_per_pos + (i_11 * 4)) = __7; - uint2 __9; - float4 v__7 = *(float4*)(ol + (i_11 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__9))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__9))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __9; - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_12 = 0; i_12 < 8; ++i_12) { - *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __10; - uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4)); - ((float2*)(&__10))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]); - ((float2*)(&__10))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]); - *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __10; - } - } - #pragma unroll - for (int i_13 = 0; i_13 < 4; ++i_13) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)]; - #pragma unroll - for (int i_14 = 0; i_14 < 16; ++i_14) { - ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)])); - } - } - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - float4 __11; - float4 v__9 = *(float4*)(sumsq_per_pos + (i_15 * 4)); - float4 __12; - float4 v__10 = *(float4*)(ol + (i_15 * 4)); - __12.x = (v__10.x*v__10.x); - __12.y = (v__10.y*v__10.y); - __12.z = (v__10.z*v__10.z); - __12.w = (v__10.w*v__10.w); - __11.x = (v__9.x+__12.x); - __11.y = (v__9.y+__12.y); - __11.z = (v__9.z+__12.z); - __11.w = (v__9.w+__12.w); - *(float4*)(sumsq_per_pos + (i_15 * 4)) = __11; - uint2 __13; - float4 v__11 = *(float4*)(ol + (i_15 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__13))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__13))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __13; - } - sumsq[0] = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int rv = 0; rv < 16; ++rv) { - sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]); - } - tl::__sync_thread_partial<3, 64>(); - sumsq[0] = tl::AllReduce>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192]))); - rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_17 = 0; i_17 < 2; ++i_17) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)]))); - } - tl::cp_async_commit(); - for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_18 = 0; i_18 < 2; ++i_18) { - *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_3 = 0; vec_3 < 2; ++vec_3) { - float4 __14; - uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4)); - ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]); - ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]); - *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __14; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_19 = 0; i_19 < 2; ++i_19) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)]))); - } - tl::cp_async_commit(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_20 = 0; i_20 < 2; ++i_20) { - *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984)); - for (int vec_4 = 0; vec_4 < 2; ++vec_4) { - float4 __15; - float4 __16; - float4 __17; - uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4)); - ((float2*)(&__17))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]); - ((float2*)(&__17))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]); - float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __16.x = (__17.x*v__14.x); - __16.y = (__17.y*v__14.y); - __16.z = (__17.z*v__14.z); - __16.w = (__17.w*v__14.w); - float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4))); - __15.x = (__16.x*v__15.x); - __15.y = (__16.y*v__15.y); - __15.z = (__16.z*v__15.z); - __15.w = (__16.w*v__15.w); - *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __15; - } - } - #pragma unroll - for (int i_21 = 0; i_21 < 2; ++i_21) { - for (int vec_5 = 0; vec_5 < 2; ++vec_5) { - uint2 __18; - float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__18))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__18))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]); - *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __18; - } - *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0); - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_22 = 0; i_22 < 2; ++i_22) { - *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_6 = 0; vec_6 < 2; ++vec_6) { - float4 __19; - uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4)); - ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]); - ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]); - *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __19; - } - } - #pragma unroll - for (int i_23 = 0; i_23 < 2; ++i_23) { - *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032)); - for (int vec_7 = 0; vec_7 < 2; ++vec_7) { - float4 __20; - float4 __21; - float4 __22; - uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4)); - ((float2*)(&__22))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]); - ((float2*)(&__22))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]); - float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __21.x = (__22.x*v__19.x); - __21.y = (__22.y*v__19.y); - __21.z = (__22.z*v__19.z); - __21.w = (__22.w*v__19.w); - float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4))); - __20.x = (__21.x*v__20.x); - __20.y = (__21.y*v__20.y); - __20.z = (__21.z*v__20.z); - __20.w = (__21.w*v__20.w); - *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __20; - } - } - #pragma unroll - for (int i_24 = 0; i_24 < 2; ++i_24) { - for (int vec_8 = 0; vec_8 < 2; ++vec_8) { - uint2 __23; - float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__23))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__23))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]); - *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __23; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0); - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_25 = 0; i_25 < 2; ++i_25) { - *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104)); - for (int vec_9 = 0; vec_9 < 2; ++vec_9) { - float4 __24; - uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4)); - ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]); - ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]); - *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __24; - } - } - #pragma unroll - for (int i_26 = 0; i_26 < 2; ++i_26) { - *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056)); - for (int vec_10 = 0; vec_10 < 2; ++vec_10) { - float4 __25; - float4 __26; - float4 __27; - uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4)); - ((float2*)(&__27))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]); - ((float2*)(&__27))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]); - float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __26.x = (__27.x*v__24.x); - __26.y = (__27.y*v__24.y); - __26.z = (__27.z*v__24.z); - __26.w = (__27.w*v__24.w); - float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4))); - __25.x = (__26.x*v__25.x); - __25.y = (__26.y*v__25.y); - __25.z = (__26.z*v__25.z); - __25.w = (__26.w*v__25.w); - *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __25; - } - } - #pragma unroll - for (int i_27 = 0; i_27 < 2; ++i_27) { - for (int vec_11 = 0; vec_11 < 2; ++vec_11) { - uint2 __28; - float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__28))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__28))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]); - *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __28; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f417b351704372ae9697b314cec4913d9e69469d246e2c9d2c8400a16396625/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f417b351704372ae9697b314cec4913d9e69469d246e2c9d2c8400a16396625/host_kernel.cu deleted file mode 100644 index 13dbe06..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f417b351704372ae9697b314cec4913d9e69469d246e2c9d2c8400a16396625/host_kernel.cu +++ /dev/null @@ -1,3359 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_pre_big_fuse_with_norm_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[16]; - void* stack_ffi_any = stack; - if (!((num_args == 9))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_pre_big_fuse_with_norm_tilelang: num_args should be 9", (long long)(num_args), (long long)(9)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang: args pointer is NULL"); - return -1; - } - int32_t gemm_out_mul_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((gemm_out_mul_type_index == 0) || (gemm_out_mul_type_index == 4)) || (gemm_out_mul_type_index == 7)) || (64 <= gemm_out_mul_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_mul expected pointer or tensor handle"); - return -1; - } - int32_t gemm_out_sqrsum_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((gemm_out_sqrsum_type_index == 0) || (gemm_out_sqrsum_type_index == 4)) || (gemm_out_sqrsum_type_index == 7)) || (64 <= gemm_out_sqrsum_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_sqrsum expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t layer_input_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((layer_input_type_index == 0) || (layer_input_type_index == 4)) || (layer_input_type_index == 7)) || (64 <= layer_input_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input layer_input expected pointer or tensor handle"); - return -1; - } - int32_t norm_weight_type_index = (((TVMFFIAny*)args)[8].type_index); - if (!(((((norm_weight_type_index == 0) || (norm_weight_type_index == 4)) || (norm_weight_type_index == 7)) || (64 <= norm_weight_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input norm_weight expected pointer or tensor handle"); - return -1; - } - void* gemm_out_mul = ((gemm_out_mul_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* gemm_out_sqrsum = ((gemm_out_sqrsum_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* layer_input = ((layer_input_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - void* norm_weight = ((norm_weight_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[8].v_ptr) + 24)) : (((TVMFFIAny*)args)[8].v_ptr)); - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null = (gemm_out_mul == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null = (gemm_out_sqrsum == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_residual_is_null = (residual == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null = (layer_input == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.layer_input is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null = (norm_weight == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.norm_weight is expected to have non-NULL pointer"); - return -1; - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape = (((DLTensor*)gemm_out_mul)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape = (((DLTensor*)gemm_out_sqrsum)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape = (((DLTensor*)layer_input)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape = (((DLTensor*)norm_weight)[0].shape); - if ((((DLTensor*)gemm_out_mul)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul, mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum, mhc_pre_big_fuse_with_norm_tilelang.residual, mhc_pre_big_fuse_with_norm_tilelang.post_mix, mhc_pre_big_fuse_with_norm_tilelang.comb_mix, mhc_pre_big_fuse_with_norm_tilelang.layer_input"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides = (((DLTensor*)gemm_out_mul)[0].strides); - int32_t dev_id = (((DLTensor*)gemm_out_mul)[0].device.device_id); - void* gemm_out_mul_1 = (((DLTensor*)gemm_out_mul)[0].data); - if ((((DLTensor*)gemm_out_sqrsum)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides = (((DLTensor*)gemm_out_sqrsum)[0].strides); - void* gemm_out_sqrsum_1 = (((DLTensor*)gemm_out_sqrsum)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)comb_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)layer_input)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides = (((DLTensor*)layer_input)[0].strides); - void* layer_input_1 = (((DLTensor*)layer_input)[0].data); - if ((((DLTensor*)norm_weight)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_9) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides = (((DLTensor*)norm_weight)[0].strides); - void* norm_weight_1 = (((DLTensor*)norm_weight)[0].data); - if ((((((DLTensor*)gemm_out_mul)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_mul)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_mul)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0]) != 6) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)6; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[1]); - } - if ((condval_2 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - int32_t condval_3; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[0]); - } - if (((num_tokens * 24) != condval_3) && (num_tokens != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_15) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_mul)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_mul)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_17) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_18) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)gemm_out_sqrsum)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0]) != 6) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)6; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[1]); - } - if ((condval_4 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_22) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[0]); - } - if ((((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != condval_5) && (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_23) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_sqrsum)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_26) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_27) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_28) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_29) != 0) { - return -1; - } - } - int32_t condval_6; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - if (condval_6 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_7; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_7); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_34) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_35) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_36) != 0) { - return -1; - } - } - int32_t condval_8; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - if (condval_8 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_9; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_9); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_38) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_40) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_41) != 0) { - return -1; - } - } - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_42) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_46) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - if (condval_12 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_13; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_13); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_47) != 0) { - return -1; - } - } - int32_t condval_14; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[0]); - } - if ((condval_14 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_48) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_49) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_51) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_52) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_53) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_54) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - if (condval_15 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_16; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_16); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_56) != 0) { - return -1; - } - } - int32_t condval_17; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[0]); - } - if ((condval_17 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_58) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_60) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_61) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_62) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_63) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1]) != 16) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_64) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - if (condval_18 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[0]); - } - if ((condval_20 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_66) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_67) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_68) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_69) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_70) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)layer_input)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)layer_input)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)layer_input)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_71) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_72) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_73) != 0) { - return -1; - } - } - int32_t condval_21; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - if (condval_21 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_22; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_22); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - int32_t condval_23; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[0]); - } - if ((condval_23 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_75) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)layer_input)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_76) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_77) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_78) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_79) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)norm_weight)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)norm_weight)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)norm_weight)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_80) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - if (condval_24 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_25; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_25); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)norm_weight)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_83) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_85) != 0) { - return -1; - } - } - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_86) != 0) { - return -1; - } - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_87) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = gemm_out_mul_1; - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = gemm_out_sqrsum_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = hc_scale_1; - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = layer_input_1; - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = norm_weight_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = post_mix_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)96); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)29040); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = (int64_t)0; - if (mhc_pre_big_fuse_with_norm_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_pre_big_fuse_with_norm_tilelang_kernel", &mhc_pre_big_fuse_with_norm_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_pre_big_fuse_with_norm_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 15, &result_88) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f417b351704372ae9697b314cec4913d9e69469d246e2c9d2c8400a16396625/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f417b351704372ae9697b314cec4913d9e69469d246e2c9d2c8400a16396625/params.pkl deleted file mode 100644 index 715ab644739bb72d69b424a89aef8369fbc7d3bd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2200 zcmeHI&r2IY6jozmKq~<$YN24~6d^4BfF5%8HdMrlmm+IQBzp zV%04(1wWY%&!J-xTowwfGyzmz4BiO(a`#FtrcYih4(g->fU-FTi_56iXM{W@8z$%qzd<8cMb5s4Wa t5MBH>*O+=r&Dpc--K*kyebY_(;=g*|!{Uv4^;vmk3*YV*CgYU{!C!}A_6-04 diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f58ab0397f1bec57393c246b40519995302f81a4f075621dbc500f561f77d46/device_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f58ab0397f1bec57393c246b40519995302f81a4f075621dbc500f561f77d46/device_kernel.cu deleted file mode 100644 index 6c55a01..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f58ab0397f1bec57393c246b40519995302f81a4f075621dbc500f561f77d46/device_kernel.cu +++ /dev/null @@ -1,424 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float mixes[1]; - float rms[1]; - float cm[1]; - float row_max[1]; - float row_sum[1]; - float col_sum[1]; - float sumsq_per_pos[16]; - float sumsq[1]; - float rsqrt_norm[1]; - float xl[64]; - float ol[16]; - bfloat16_t xs_local_cast[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t xs_local_cast_2[8]; - float w_local[16]; - float ol_1[16]; - bfloat16_t w_shared_local_cast_3[8]; - bfloat16_t output_shared_local_cast_4[8]; - bfloat16_t layer_input_local_cast_5[8]; - bfloat16_t w_shared_local_cast_6[8]; - bfloat16_t output_shared_local_cast_7[8]; - bfloat16_t layer_input_local_cast_8[8]; - bfloat16_t w_shared_local_cast_9[8]; - bfloat16_t output_shared_local_cast_10[8]; - bfloat16_t layer_input_local_cast_11[8]; - mixes[0] = 0x0p+0f/*0.000000e+00*/; - rms[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - for (int i_split = 0; i_split < 15; ++i_split) { - rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]); - } - rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - mixes[0] = 0x0p+0f/*0.000000e+00*/; - for (int i_split_1 = 0; i_split_1 < 15; ++i_split_1) { - mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]); - } - mixes[0] = (mixes[0] * rms[0]); - if ((((int)threadIdx.x) / 24) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0]; - } - __syncthreads(); - if (((int)threadIdx.x) < 32) { - if (((int)threadIdx.x) < 4) { - post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/); - } - cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]); - row_max[0] = -CUDART_INF_F; - row_max[0] = max(row_max[0], cm[0]); - row_max[0] = tl::AllReduce>::run(row_max[0]); - cm[0] = expf((cm[0] - row_max[0])); - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - for (int __1 = 0; __1 < 19; ++__1) { - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - } - if ((((int)threadIdx.x) >> 4) == 0) { - comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0]; - } - } else { - if (((int)threadIdx.x) < 36) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - #pragma unroll - for (int i = 0; i < 4; ++i) { - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_2 = 0; i_2 < 8; ++i_2) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_4 = 0; i_4 < 8; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - tl::__sync_thread_partial<3, 64>(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)]; - #pragma unroll - for (int i_6 = 0; i_6 < 16; ++i_6) { - ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)])); - } - } - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - float4 __3; - float4 v__1 = *(float4*)(sumsq_per_pos + (i_7 * 4)); - float4 __4; - float4 v__2 = *(float4*)(ol + (i_7 * 4)); - __4.x = (v__2.x*v__2.x); - __4.y = (v__2.y*v__2.y); - __4.z = (v__2.z*v__2.z); - __4.w = (v__2.w*v__2.w); - __3.x = (v__1.x+__4.x); - __3.y = (v__1.y+__4.y); - __3.z = (v__1.z+__4.z); - __3.w = (v__1.w+__4.w); - *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3; - uint2 __5; - float4 v__3 = *(float4*)(ol + (i_7 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __5; - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_8 = 0; i_8 < 8; ++i_8) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __6; - uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]); - *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __6; - } - } - #pragma unroll - for (int i_9 = 0; i_9 < 4; ++i_9) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)]; - #pragma unroll - for (int i_10 = 0; i_10 < 16; ++i_10) { - ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)])); - } - } - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - float4 __7; - float4 v__5 = *(float4*)(sumsq_per_pos + (i_11 * 4)); - float4 __8; - float4 v__6 = *(float4*)(ol + (i_11 * 4)); - __8.x = (v__6.x*v__6.x); - __8.y = (v__6.y*v__6.y); - __8.z = (v__6.z*v__6.z); - __8.w = (v__6.w*v__6.w); - __7.x = (v__5.x+__8.x); - __7.y = (v__5.y+__8.y); - __7.z = (v__5.z+__8.z); - __7.w = (v__5.w+__8.w); - *(float4*)(sumsq_per_pos + (i_11 * 4)) = __7; - uint2 __9; - float4 v__7 = *(float4*)(ol + (i_11 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__9))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__9))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __9; - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_12 = 0; i_12 < 8; ++i_12) { - *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __10; - uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4)); - ((float2*)(&__10))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]); - ((float2*)(&__10))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]); - *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __10; - } - } - #pragma unroll - for (int i_13 = 0; i_13 < 4; ++i_13) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)]; - #pragma unroll - for (int i_14 = 0; i_14 < 16; ++i_14) { - ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)])); - } - } - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - float4 __11; - float4 v__9 = *(float4*)(sumsq_per_pos + (i_15 * 4)); - float4 __12; - float4 v__10 = *(float4*)(ol + (i_15 * 4)); - __12.x = (v__10.x*v__10.x); - __12.y = (v__10.y*v__10.y); - __12.z = (v__10.z*v__10.z); - __12.w = (v__10.w*v__10.w); - __11.x = (v__9.x+__12.x); - __11.y = (v__9.y+__12.y); - __11.z = (v__9.z+__12.z); - __11.w = (v__9.w+__12.w); - *(float4*)(sumsq_per_pos + (i_15 * 4)) = __11; - uint2 __13; - float4 v__11 = *(float4*)(ol + (i_15 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__13))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__13))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __13; - } - sumsq[0] = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int rv = 0; rv < 16; ++rv) { - sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]); - } - tl::__sync_thread_partial<3, 64>(); - sumsq[0] = tl::AllReduce>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192]))); - rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_17 = 0; i_17 < 2; ++i_17) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)]))); - } - tl::cp_async_commit(); - for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_18 = 0; i_18 < 2; ++i_18) { - *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_3 = 0; vec_3 < 2; ++vec_3) { - float4 __14; - uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4)); - ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]); - ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]); - *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __14; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_19 = 0; i_19 < 2; ++i_19) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)]))); - } - tl::cp_async_commit(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_20 = 0; i_20 < 2; ++i_20) { - *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984)); - for (int vec_4 = 0; vec_4 < 2; ++vec_4) { - float4 __15; - float4 __16; - float4 __17; - uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4)); - ((float2*)(&__17))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]); - ((float2*)(&__17))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]); - float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __16.x = (__17.x*v__14.x); - __16.y = (__17.y*v__14.y); - __16.z = (__17.z*v__14.z); - __16.w = (__17.w*v__14.w); - float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4))); - __15.x = (__16.x*v__15.x); - __15.y = (__16.y*v__15.y); - __15.z = (__16.z*v__15.z); - __15.w = (__16.w*v__15.w); - *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __15; - } - } - #pragma unroll - for (int i_21 = 0; i_21 < 2; ++i_21) { - for (int vec_5 = 0; vec_5 < 2; ++vec_5) { - uint2 __18; - float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__18))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__18))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]); - *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __18; - } - *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0); - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_22 = 0; i_22 < 2; ++i_22) { - *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_6 = 0; vec_6 < 2; ++vec_6) { - float4 __19; - uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4)); - ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]); - ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]); - *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __19; - } - } - #pragma unroll - for (int i_23 = 0; i_23 < 2; ++i_23) { - *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032)); - for (int vec_7 = 0; vec_7 < 2; ++vec_7) { - float4 __20; - float4 __21; - float4 __22; - uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4)); - ((float2*)(&__22))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]); - ((float2*)(&__22))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]); - float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __21.x = (__22.x*v__19.x); - __21.y = (__22.y*v__19.y); - __21.z = (__22.z*v__19.z); - __21.w = (__22.w*v__19.w); - float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4))); - __20.x = (__21.x*v__20.x); - __20.y = (__21.y*v__20.y); - __20.z = (__21.z*v__20.z); - __20.w = (__21.w*v__20.w); - *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __20; - } - } - #pragma unroll - for (int i_24 = 0; i_24 < 2; ++i_24) { - for (int vec_8 = 0; vec_8 < 2; ++vec_8) { - uint2 __23; - float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__23))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__23))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]); - *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __23; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0); - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_25 = 0; i_25 < 2; ++i_25) { - *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104)); - for (int vec_9 = 0; vec_9 < 2; ++vec_9) { - float4 __24; - uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4)); - ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]); - ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]); - *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __24; - } - } - #pragma unroll - for (int i_26 = 0; i_26 < 2; ++i_26) { - *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056)); - for (int vec_10 = 0; vec_10 < 2; ++vec_10) { - float4 __25; - float4 __26; - float4 __27; - uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4)); - ((float2*)(&__27))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]); - ((float2*)(&__27))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]); - float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __26.x = (__27.x*v__24.x); - __26.y = (__27.y*v__24.y); - __26.z = (__27.z*v__24.z); - __26.w = (__27.w*v__24.w); - float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4))); - __25.x = (__26.x*v__25.x); - __25.y = (__26.y*v__25.y); - __25.z = (__26.z*v__25.z); - __25.w = (__26.w*v__25.w); - *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __25; - } - } - #pragma unroll - for (int i_27 = 0; i_27 < 2; ++i_27) { - for (int vec_11 = 0; vec_11 < 2; ++vec_11) { - uint2 __28; - float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__28))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__28))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]); - *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __28; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f58ab0397f1bec57393c246b40519995302f81a4f075621dbc500f561f77d46/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f58ab0397f1bec57393c246b40519995302f81a4f075621dbc500f561f77d46/host_kernel.cu deleted file mode 100644 index e209611..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f58ab0397f1bec57393c246b40519995302f81a4f075621dbc500f561f77d46/host_kernel.cu +++ /dev/null @@ -1,3359 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_pre_big_fuse_with_norm_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[16]; - void* stack_ffi_any = stack; - if (!((num_args == 9))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_pre_big_fuse_with_norm_tilelang: num_args should be 9", (long long)(num_args), (long long)(9)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang: args pointer is NULL"); - return -1; - } - int32_t gemm_out_mul_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((gemm_out_mul_type_index == 0) || (gemm_out_mul_type_index == 4)) || (gemm_out_mul_type_index == 7)) || (64 <= gemm_out_mul_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_mul expected pointer or tensor handle"); - return -1; - } - int32_t gemm_out_sqrsum_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((gemm_out_sqrsum_type_index == 0) || (gemm_out_sqrsum_type_index == 4)) || (gemm_out_sqrsum_type_index == 7)) || (64 <= gemm_out_sqrsum_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_sqrsum expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t layer_input_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((layer_input_type_index == 0) || (layer_input_type_index == 4)) || (layer_input_type_index == 7)) || (64 <= layer_input_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input layer_input expected pointer or tensor handle"); - return -1; - } - int32_t norm_weight_type_index = (((TVMFFIAny*)args)[8].type_index); - if (!(((((norm_weight_type_index == 0) || (norm_weight_type_index == 4)) || (norm_weight_type_index == 7)) || (64 <= norm_weight_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input norm_weight expected pointer or tensor handle"); - return -1; - } - void* gemm_out_mul = ((gemm_out_mul_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* gemm_out_sqrsum = ((gemm_out_sqrsum_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* layer_input = ((layer_input_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - void* norm_weight = ((norm_weight_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[8].v_ptr) + 24)) : (((TVMFFIAny*)args)[8].v_ptr)); - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null = (gemm_out_mul == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null = (gemm_out_sqrsum == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_residual_is_null = (residual == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null = (layer_input == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.layer_input is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null = (norm_weight == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.norm_weight is expected to have non-NULL pointer"); - return -1; - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape = (((DLTensor*)gemm_out_mul)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape = (((DLTensor*)gemm_out_sqrsum)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape = (((DLTensor*)layer_input)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape = (((DLTensor*)norm_weight)[0].shape); - if ((((DLTensor*)gemm_out_mul)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul, mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum, mhc_pre_big_fuse_with_norm_tilelang.residual, mhc_pre_big_fuse_with_norm_tilelang.post_mix, mhc_pre_big_fuse_with_norm_tilelang.comb_mix, mhc_pre_big_fuse_with_norm_tilelang.layer_input"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides = (((DLTensor*)gemm_out_mul)[0].strides); - int32_t dev_id = (((DLTensor*)gemm_out_mul)[0].device.device_id); - void* gemm_out_mul_1 = (((DLTensor*)gemm_out_mul)[0].data); - if ((((DLTensor*)gemm_out_sqrsum)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides = (((DLTensor*)gemm_out_sqrsum)[0].strides); - void* gemm_out_sqrsum_1 = (((DLTensor*)gemm_out_sqrsum)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)comb_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)layer_input)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides = (((DLTensor*)layer_input)[0].strides); - void* layer_input_1 = (((DLTensor*)layer_input)[0].data); - if ((((DLTensor*)norm_weight)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_9) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides = (((DLTensor*)norm_weight)[0].strides); - void* norm_weight_1 = (((DLTensor*)norm_weight)[0].data); - if ((((((DLTensor*)gemm_out_mul)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_mul)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_mul)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0]) != 15) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)15; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[1]); - } - if ((condval_2 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - int32_t condval_3; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[0]); - } - if (((num_tokens * 24) != condval_3) && (num_tokens != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_15) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_mul)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_mul)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_17) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_18) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)gemm_out_sqrsum)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0]) != 15) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)15; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[1]); - } - if ((condval_4 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_22) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[0]); - } - if ((((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != condval_5) && (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_23) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_sqrsum)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_26) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_27) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_28) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_29) != 0) { - return -1; - } - } - int32_t condval_6; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - if (condval_6 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_7; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_7); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_34) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_35) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_36) != 0) { - return -1; - } - } - int32_t condval_8; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - if (condval_8 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_9; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_9); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_38) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_40) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_41) != 0) { - return -1; - } - } - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_42) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_46) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - if (condval_12 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_13; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_13); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_47) != 0) { - return -1; - } - } - int32_t condval_14; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[0]); - } - if ((condval_14 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_48) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_49) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_51) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_52) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_53) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_54) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - if (condval_15 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_16; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_16); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_56) != 0) { - return -1; - } - } - int32_t condval_17; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[0]); - } - if ((condval_17 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_58) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_60) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_61) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_62) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_63) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1]) != 16) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_64) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - if (condval_18 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[0]); - } - if ((condval_20 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_66) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_67) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_68) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_69) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_70) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)layer_input)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)layer_input)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)layer_input)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_71) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_72) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_73) != 0) { - return -1; - } - } - int32_t condval_21; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - if (condval_21 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_22; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_22); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - int32_t condval_23; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[0]); - } - if ((condval_23 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_75) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)layer_input)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_76) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_77) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_78) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_79) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)norm_weight)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)norm_weight)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)norm_weight)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_80) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - if (condval_24 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_25; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_25); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)norm_weight)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_83) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_85) != 0) { - return -1; - } - } - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_86) != 0) { - return -1; - } - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_87) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = gemm_out_mul_1; - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = gemm_out_sqrsum_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = hc_scale_1; - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = layer_input_1; - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = norm_weight_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = post_mix_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)96); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)29040); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = (int64_t)0; - if (mhc_pre_big_fuse_with_norm_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_pre_big_fuse_with_norm_tilelang_kernel", &mhc_pre_big_fuse_with_norm_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_pre_big_fuse_with_norm_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 15, &result_88) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f58ab0397f1bec57393c246b40519995302f81a4f075621dbc500f561f77d46/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/1f58ab0397f1bec57393c246b40519995302f81a4f075621dbc500f561f77d46/params.pkl deleted file mode 100644 index 91fc240dad75a28fd194a82c213e24bed6030440..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2200 zcmeHI&r2IY6xPPXfYt^|sTB%#PAP61{stx2xVL$H?|t9<-oBZC)$6PJT=|{&ru$@N3^7-|08~#Z2vTl%;@uJj zqafZdPecFx+~+IQBz7 zh!wZQ6r3_0{(_E0a9Jp@(gbjp8t*E{++g?uvr%MCa1|k=yBQFpwPjL8a7Nl&EuwS; z8EO9MQ!hu6;r0S6tiBKM5R3CfLzd>d9kx0I8WKzh#g$_uTaz)xd zFn3HK75Yc=dlZFi)cbN0%H;`)+-Q|*8CObMZly+s%{)KJeHPMH=!c>*8m6d;`jeT7 z_oC4hA0PTFK98JT6JJNBiEp#u*3@3n3ZK?yy81Y^9+qphu{~L<$%qzc<8B4Y5s4Wa t5M9`vYfL?*=Iq(^=2daMzUroY@?X8}Vev*i`>Z^(g|BxDlkv)f;6DaP_8$NM diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2271822608992f393db54113760d3441caf5e37b0ed41e8e367fb0b64d28f248/device_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2271822608992f393db54113760d3441caf5e37b0ed41e8e367fb0b64d28f248/device_kernel.cu deleted file mode 100644 index cf5ce59..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2271822608992f393db54113760d3441caf5e37b0ed41e8e367fb0b64d28f248/device_kernel.cu +++ /dev/null @@ -1,74 +0,0 @@ -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_post_tilelang_kernel(const float* a, const bfloat16_t* b, const float* c, const bfloat16_t* d, bfloat16_t* x, int num_tokens); -extern "C" __global__ void __launch_bounds__(128, 1) mhc_post_tilelang_kernel(const float* a, const bfloat16_t* b, const float* c, const bfloat16_t* d, bfloat16_t* x, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float a_local[16]; - float c_local[4]; - float b_local[32]; - bfloat16_t b_shared_local_cast[8]; - bfloat16_t d_shared_local_cast_1[8]; - float d_local[8]; - float x_local[32]; - bfloat16_t x_local_cast_2[8]; - cudaGridDependencySynchronize(); - #pragma unroll - for (int i = 0; i < 4; ++i) { - *(float4*)(a_local + (i * 4)) = *(float4*)(a + ((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)i) * (int64_t)4))); - } - *(float4*)(c_local + 0) = *(float4*)(c + (((int64_t)((int)blockIdx.x)) * (int64_t)4)); - for (int i0_h = 0; i0_h < 4; ++i0_h) { - #pragma unroll - for (int i_1 = 0; i_1 < 4; ++i_1) { - *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((i_1 * 1024) + (((int)threadIdx.x) * 8))) = *(uint4*)(b + ((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + (((int64_t)i_1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8))); - } - *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((int)threadIdx.x) * 8) + 4096)) = *(uint4*)(d + (((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h) * (int64_t)1024)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8))); - #pragma unroll - for (int i_2 = 0; i_2 < 4; ++i_2) { - *(uint4*)(b_shared_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((i_2 * 1024) + (((int)threadIdx.x) * 8))); - for (int vec = 0; vec < 2; ++vec) { - float4 __1; - uint2 v_ = *(uint2*)(b_shared_local_cast + (vec * 4)); - ((float2*)(&__1))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__1))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(b_local + ((i_2 * 8) + (vec * 4))) = __1; - } - } - *(uint4*)(d_shared_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((int)threadIdx.x) * 8) + 4096)); - for (int i_3 = 0; i_3 < 2; ++i_3) { - float4 __2; - uint2 v__1 = *(uint2*)(d_shared_local_cast_1 + (i_3 * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__1))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__1))[1]); - *(float4*)(d_local + (i_3 * 4)) = __2; - } - #pragma unroll - for (int i_4 = 0; i_4 < 32; ++i_4) { - x_local[i_4] = (c_local[(i_4 >> 3)] * d_local[(i_4 & 7)]); - for (int i_hci = 0; i_hci < 4; ++i_hci) { - x_local[i_4] = (x_local[i_4] + (a_local[((i_hci * 4) + (i_4 >> 3))] * b_local[((i_hci * 8) + (i_4 & 7))])); - } - } - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - uint2 __3; - float4 v__2 = *(float4*)(x_local + ((i_5 * 8) + (vec_1 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__3))[0] = __float22bfloat162_rn(((float2*)(&v__2))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__3))[1] = __float22bfloat162_rn(((float2*)(&v__2))[1]); - *(uint2*)(x_local_cast_2 + (vec_1 * 4)) = __3; - } - *(uint4*)(x + ((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + (((int64_t)i_5) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8))) = *(uint4*)(x_local_cast_2 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2271822608992f393db54113760d3441caf5e37b0ed41e8e367fb0b64d28f248/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2271822608992f393db54113760d3441caf5e37b0ed41e8e367fb0b64d28f248/host_kernel.cu deleted file mode 100644 index 94bb238..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2271822608992f393db54113760d3441caf5e37b0ed41e8e367fb0b64d28f248/host_kernel.cu +++ /dev/null @@ -1,2176 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_post_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_post_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_post_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[12]; - void* stack_ffi_any = stack; - if (!((num_args == 5))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_post_tilelang: num_args should be 5", (long long)(num_args), (long long)(5)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_post_tilelang: args pointer is NULL"); - return -1; - } - int32_t a_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((a_type_index == 0) || (a_type_index == 4)) || (a_type_index == 7)) || (64 <= a_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_post_tilelang input a expected pointer or tensor handle"); - return -1; - } - int32_t b_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((b_type_index == 0) || (b_type_index == 4)) || (b_type_index == 7)) || (64 <= b_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_post_tilelang input b expected pointer or tensor handle"); - return -1; - } - int32_t c_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((c_type_index == 0) || (c_type_index == 4)) || (c_type_index == 7)) || (64 <= c_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_post_tilelang input c expected pointer or tensor handle"); - return -1; - } - int32_t d_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((d_type_index == 0) || (d_type_index == 4)) || (d_type_index == 7)) || (64 <= d_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_post_tilelang input d expected pointer or tensor handle"); - return -1; - } - int32_t x_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((x_type_index == 0) || (x_type_index == 4)) || (x_type_index == 7)) || (64 <= x_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_post_tilelang input x expected pointer or tensor handle"); - return -1; - } - void* a = ((a_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* b = ((b_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* c = ((c_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* d = ((d_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* x = ((x_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - bool mhc_post_tilelang_a_is_null = (a == NULL); - if (!(!mhc_post_tilelang_a_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_post_tilelang.a is expected to have non-NULL pointer"); - return -1; - } - bool mhc_post_tilelang_b_is_null = (b == NULL); - if (!(!mhc_post_tilelang_b_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_post_tilelang.b is expected to have non-NULL pointer"); - return -1; - } - bool mhc_post_tilelang_c_is_null = (c == NULL); - if (!(!mhc_post_tilelang_c_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_post_tilelang.c is expected to have non-NULL pointer"); - return -1; - } - bool mhc_post_tilelang_d_is_null = (d == NULL); - if (!(!mhc_post_tilelang_d_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_post_tilelang.d is expected to have non-NULL pointer"); - return -1; - } - bool mhc_post_tilelang_x_is_null = (x == NULL); - if (!(!mhc_post_tilelang_x_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_post_tilelang.x is expected to have non-NULL pointer"); - return -1; - } - void* mhc_post_tilelang_a_shape = (((DLTensor*)a)[0].shape); - void* mhc_post_tilelang_b_shape = (((DLTensor*)b)[0].shape); - void* mhc_post_tilelang_c_shape = (((DLTensor*)c)[0].shape); - void* mhc_post_tilelang_d_shape = (((DLTensor*)d)[0].shape); - void* mhc_post_tilelang_x_shape = (((DLTensor*)x)[0].shape); - if ((((DLTensor*)a)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"a"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)a)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_post_tilelang.a, mhc_post_tilelang.b, mhc_post_tilelang.c, mhc_post_tilelang.d, mhc_post_tilelang.x"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_post_tilelang_a_shape)[0]); - void* mhc_post_tilelang_a_strides = (((DLTensor*)a)[0].strides); - int32_t dev_id = (((DLTensor*)a)[0].device.device_id); - void* a_1 = (((DLTensor*)a)[0].data); - if ((((DLTensor*)b)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"b"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)b)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_post_tilelang_b_strides = (((DLTensor*)b)[0].strides); - void* b_1 = (((DLTensor*)b)[0].data); - if ((((DLTensor*)c)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"c"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)c)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_post_tilelang_c_strides = (((DLTensor*)c)[0].strides); - void* c_1 = (((DLTensor*)c)[0].data); - if ((((DLTensor*)d)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"d"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)d)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_post_tilelang_d_strides = (((DLTensor*)d)[0].strides); - void* d_1 = (((DLTensor*)d)[0].data); - if ((((DLTensor*)x)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)x)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_post_tilelang_x_strides = (((DLTensor*)x)[0].strides); - void* x_1 = (((DLTensor*)x)[0].data); - if ((((((DLTensor*)a)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)a)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)a)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"a"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)a)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)a)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)a)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_6) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_post_tilelang_a_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"a"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_post_tilelang_a_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_7) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_post_tilelang_a_shape)[2]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"a"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_post_tilelang_a_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_8) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_post_tilelang_a_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_post_tilelang_a_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"a"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_post_tilelang_a_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_post_tilelang_a_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_9) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_post_tilelang_a_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_post_tilelang_a_strides)[1]); - } - if (condval_2 != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"a"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_3; - if ((mhc_post_tilelang_a_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_post_tilelang_a_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_3); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_10) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_post_tilelang_a_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_post_tilelang_a_strides)[0]); - } - if ((condval_4 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"a"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_11) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)a)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"a"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)a)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_12) != 0) { - return -1; - } - } - if ((((DLTensor*)a)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"a"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)a)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_13) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (a_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"a"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)b)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)b)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)b)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"b"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)b)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)b)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)b)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_15) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_post_tilelang_a_shape)[0]) != ((int32_t)((int64_t*)mhc_post_tilelang_b_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"b"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_post_tilelang_b_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_post_tilelang_a_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_16) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_post_tilelang_b_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"b"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_post_tilelang_b_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_17) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_post_tilelang_b_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"b"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_post_tilelang_b_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_18) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_post_tilelang_b_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_post_tilelang_b_strides)[2]); - } - if (condval_5 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"b"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_6; - if ((mhc_post_tilelang_b_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_post_tilelang_b_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_6); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_19) != 0) { - return -1; - } - } - int32_t condval_7; - if ((mhc_post_tilelang_b_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_post_tilelang_b_strides)[1]); - } - if (condval_7 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"b"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_8; - if ((mhc_post_tilelang_b_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_post_tilelang_b_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_8); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - int32_t condval_9; - if ((mhc_post_tilelang_b_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_post_tilelang_b_strides)[0]); - } - if ((condval_9 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"b"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_21) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)b)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"b"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)b)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_22) != 0) { - return -1; - } - } - if ((((DLTensor*)b)[0].device.device_id) != (((DLTensor*)a)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"b"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)a)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)b)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_23) != 0) { - return -1; - } - } - if ((((DLTensor*)b)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"b"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)b)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (b_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"b"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_25) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)c)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)c)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)c)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"c"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)c)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)c)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)c)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_26) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_post_tilelang_a_shape)[0]) != ((int32_t)((int64_t*)mhc_post_tilelang_c_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"c"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_post_tilelang_c_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_post_tilelang_a_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_27) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_post_tilelang_c_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"c"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_post_tilelang_c_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_28) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_post_tilelang_c_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_post_tilelang_c_strides)[1]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"c"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_post_tilelang_c_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_post_tilelang_c_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_29) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_post_tilelang_c_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_post_tilelang_c_strides)[0]); - } - if ((condval_12 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"c"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_30) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)c)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"c"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)c)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if ((((DLTensor*)c)[0].device.device_id) != (((DLTensor*)a)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"c"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)a)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)c)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((((DLTensor*)c)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"c"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)c)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (c_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"c"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_34) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)d)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)d)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)d)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"d"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)d)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)d)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)d)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_35) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_post_tilelang_a_shape)[0]) != ((int32_t)((int64_t*)mhc_post_tilelang_d_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"d"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_post_tilelang_d_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_post_tilelang_a_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_36) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_post_tilelang_d_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"d"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_post_tilelang_d_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - int32_t condval_13; - if ((mhc_post_tilelang_d_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_post_tilelang_d_strides)[1]); - } - if (condval_13 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"d"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_14; - if ((mhc_post_tilelang_d_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_post_tilelang_d_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_14); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_38) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_post_tilelang_d_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_post_tilelang_d_strides)[0]); - } - if ((condval_15 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"d"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_39) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)d)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"d"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)d)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_40) != 0) { - return -1; - } - } - if ((((DLTensor*)d)[0].device.device_id) != (((DLTensor*)a)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"d"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)a)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)d)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_41) != 0) { - return -1; - } - } - if ((((DLTensor*)d)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"d"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)d)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_42) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (d_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"d"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_43) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)x)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)x)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)x)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)x)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)x)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)x)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_44) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_post_tilelang_a_shape)[0]) != ((int32_t)((int64_t*)mhc_post_tilelang_x_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_post_tilelang_x_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_post_tilelang_a_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_post_tilelang_x_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_post_tilelang_x_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_46) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_post_tilelang_x_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_post_tilelang_x_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_47) != 0) { - return -1; - } - } - int32_t condval_16; - if ((mhc_post_tilelang_x_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_post_tilelang_x_strides)[2]); - } - if (condval_16 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_17; - if ((mhc_post_tilelang_x_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_post_tilelang_x_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_17); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_48) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_post_tilelang_x_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_post_tilelang_x_strides)[1]); - } - if (condval_18 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_post_tilelang_x_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_post_tilelang_x_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_49) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_post_tilelang_x_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_post_tilelang_x_strides)[0]); - } - if ((condval_20 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_50) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)x)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)x)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_51) != 0) { - return -1; - } - } - if ((((DLTensor*)x)[0].device.device_id) != (((DLTensor*)a)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)a)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)x)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_52) != 0) { - return -1; - } - } - if ((((DLTensor*)x)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)x)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_53) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (x_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_post_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_54) != 0) { - return -1; - } - } - } else { - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_55) != 0) { - return -1; - } - if (a_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = a_1; - if (b_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = b_1; - if (c_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = c_1; - if (d_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = d_1; - if (x_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = x_1; - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = ((int64_t)128); - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)10240); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = (int64_t)0; - if (mhc_post_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_post_tilelang_kernel", &mhc_post_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_post_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 11, &result_56) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_post_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2271822608992f393db54113760d3441caf5e37b0ed41e8e367fb0b64d28f248/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2271822608992f393db54113760d3441caf5e37b0ed41e8e367fb0b64d28f248/params.pkl deleted file mode 100644 index 818f97008f1b367cae75763b41ecf0cf207d2ebd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1725 zcmeH`%Sr<=6o$RDwcv$jX%n4EhD@rJGN21xW!Ly3 zzL2ltnQ7~#C_aE@m++tb|M_xqUw(e6Ocd+VZQ3OxB8Zr5dm!6Ff|oGOVrOT4&=R8K zV%K+D4=zJ>IW%_(IJ;nOV%lm6j}8uarSI%aD^kqt?Yc+z;Epm>(V=XpvU8)drqX!J z3T#HCLzu;r99JjZ_cf}N!j zZm9~cplMKAT;my`J?JLd+39np@%RyIF=B-?8KR$k?Gde%p+bhxC5^p0kt&3gG%k81 z@0m2nk&-~N`ujfb|I}z9+oVoH42K%o!4A|l$n>)k&5yN+^YHJ|9!V1>Nn;yphXz6- zJ(7L0$YW#P--|FVi!tO*y+})$l*%w86guo?*=6R^z%ThF=mdOv@R+>9tF{^xkG8Wv zdNe-IYnyx#wY3kxd^tzE%GU`5D}FOs O|FnLj<(CG$z -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float mixes[1]; - float rms[1]; - float cm[1]; - float row_max[1]; - float row_sum[1]; - float col_sum[1]; - float sumsq_per_pos[16]; - float sumsq[1]; - float rsqrt_norm[1]; - float xl[64]; - float ol[16]; - bfloat16_t xs_local_cast[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t xs_local_cast_2[8]; - float w_local[16]; - float ol_1[16]; - bfloat16_t w_shared_local_cast_3[8]; - bfloat16_t output_shared_local_cast_4[8]; - bfloat16_t layer_input_local_cast_5[8]; - bfloat16_t w_shared_local_cast_6[8]; - bfloat16_t output_shared_local_cast_7[8]; - bfloat16_t layer_input_local_cast_8[8]; - bfloat16_t w_shared_local_cast_9[8]; - bfloat16_t output_shared_local_cast_10[8]; - bfloat16_t layer_input_local_cast_11[8]; - mixes[0] = 0x0p+0f/*0.000000e+00*/; - rms[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - for (int i_split = 0; i_split < 26; ++i_split) { - rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]); - } - rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - mixes[0] = 0x0p+0f/*0.000000e+00*/; - for (int i_split_1 = 0; i_split_1 < 26; ++i_split_1) { - mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]); - } - mixes[0] = (mixes[0] * rms[0]); - if ((((int)threadIdx.x) / 24) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0]; - } - __syncthreads(); - if (((int)threadIdx.x) < 32) { - if (((int)threadIdx.x) < 4) { - post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/); - } - cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]); - row_max[0] = -CUDART_INF_F; - row_max[0] = max(row_max[0], cm[0]); - row_max[0] = tl::AllReduce>::run(row_max[0]); - cm[0] = expf((cm[0] - row_max[0])); - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - for (int __1 = 0; __1 < 19; ++__1) { - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - } - if ((((int)threadIdx.x) >> 4) == 0) { - comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0]; - } - } else { - if (((int)threadIdx.x) < 36) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - #pragma unroll - for (int i = 0; i < 4; ++i) { - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_2 = 0; i_2 < 8; ++i_2) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_4 = 0; i_4 < 8; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - tl::__sync_thread_partial<3, 64>(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)]; - #pragma unroll - for (int i_6 = 0; i_6 < 16; ++i_6) { - ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)])); - } - } - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - float4 __3; - float4 v__1 = *(float4*)(sumsq_per_pos + (i_7 * 4)); - float4 __4; - float4 v__2 = *(float4*)(ol + (i_7 * 4)); - __4.x = (v__2.x*v__2.x); - __4.y = (v__2.y*v__2.y); - __4.z = (v__2.z*v__2.z); - __4.w = (v__2.w*v__2.w); - __3.x = (v__1.x+__4.x); - __3.y = (v__1.y+__4.y); - __3.z = (v__1.z+__4.z); - __3.w = (v__1.w+__4.w); - *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3; - uint2 __5; - float4 v__3 = *(float4*)(ol + (i_7 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __5; - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_8 = 0; i_8 < 8; ++i_8) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __6; - uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]); - *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __6; - } - } - #pragma unroll - for (int i_9 = 0; i_9 < 4; ++i_9) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)]; - #pragma unroll - for (int i_10 = 0; i_10 < 16; ++i_10) { - ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)])); - } - } - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - float4 __7; - float4 v__5 = *(float4*)(sumsq_per_pos + (i_11 * 4)); - float4 __8; - float4 v__6 = *(float4*)(ol + (i_11 * 4)); - __8.x = (v__6.x*v__6.x); - __8.y = (v__6.y*v__6.y); - __8.z = (v__6.z*v__6.z); - __8.w = (v__6.w*v__6.w); - __7.x = (v__5.x+__8.x); - __7.y = (v__5.y+__8.y); - __7.z = (v__5.z+__8.z); - __7.w = (v__5.w+__8.w); - *(float4*)(sumsq_per_pos + (i_11 * 4)) = __7; - uint2 __9; - float4 v__7 = *(float4*)(ol + (i_11 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__9))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__9))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __9; - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_12 = 0; i_12 < 8; ++i_12) { - *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __10; - uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4)); - ((float2*)(&__10))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]); - ((float2*)(&__10))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]); - *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __10; - } - } - #pragma unroll - for (int i_13 = 0; i_13 < 4; ++i_13) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)]; - #pragma unroll - for (int i_14 = 0; i_14 < 16; ++i_14) { - ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)])); - } - } - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - float4 __11; - float4 v__9 = *(float4*)(sumsq_per_pos + (i_15 * 4)); - float4 __12; - float4 v__10 = *(float4*)(ol + (i_15 * 4)); - __12.x = (v__10.x*v__10.x); - __12.y = (v__10.y*v__10.y); - __12.z = (v__10.z*v__10.z); - __12.w = (v__10.w*v__10.w); - __11.x = (v__9.x+__12.x); - __11.y = (v__9.y+__12.y); - __11.z = (v__9.z+__12.z); - __11.w = (v__9.w+__12.w); - *(float4*)(sumsq_per_pos + (i_15 * 4)) = __11; - uint2 __13; - float4 v__11 = *(float4*)(ol + (i_15 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__13))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__13))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __13; - } - sumsq[0] = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int rv = 0; rv < 16; ++rv) { - sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]); - } - tl::__sync_thread_partial<3, 64>(); - sumsq[0] = tl::AllReduce>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192]))); - rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_17 = 0; i_17 < 2; ++i_17) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)]))); - } - tl::cp_async_commit(); - for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_18 = 0; i_18 < 2; ++i_18) { - *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_3 = 0; vec_3 < 2; ++vec_3) { - float4 __14; - uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4)); - ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]); - ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]); - *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __14; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_19 = 0; i_19 < 2; ++i_19) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)]))); - } - tl::cp_async_commit(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_20 = 0; i_20 < 2; ++i_20) { - *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984)); - for (int vec_4 = 0; vec_4 < 2; ++vec_4) { - float4 __15; - float4 __16; - float4 __17; - uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4)); - ((float2*)(&__17))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]); - ((float2*)(&__17))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]); - float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __16.x = (__17.x*v__14.x); - __16.y = (__17.y*v__14.y); - __16.z = (__17.z*v__14.z); - __16.w = (__17.w*v__14.w); - float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4))); - __15.x = (__16.x*v__15.x); - __15.y = (__16.y*v__15.y); - __15.z = (__16.z*v__15.z); - __15.w = (__16.w*v__15.w); - *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __15; - } - } - #pragma unroll - for (int i_21 = 0; i_21 < 2; ++i_21) { - for (int vec_5 = 0; vec_5 < 2; ++vec_5) { - uint2 __18; - float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__18))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__18))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]); - *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __18; - } - *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0); - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_22 = 0; i_22 < 2; ++i_22) { - *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_6 = 0; vec_6 < 2; ++vec_6) { - float4 __19; - uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4)); - ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]); - ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]); - *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __19; - } - } - #pragma unroll - for (int i_23 = 0; i_23 < 2; ++i_23) { - *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032)); - for (int vec_7 = 0; vec_7 < 2; ++vec_7) { - float4 __20; - float4 __21; - float4 __22; - uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4)); - ((float2*)(&__22))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]); - ((float2*)(&__22))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]); - float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __21.x = (__22.x*v__19.x); - __21.y = (__22.y*v__19.y); - __21.z = (__22.z*v__19.z); - __21.w = (__22.w*v__19.w); - float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4))); - __20.x = (__21.x*v__20.x); - __20.y = (__21.y*v__20.y); - __20.z = (__21.z*v__20.z); - __20.w = (__21.w*v__20.w); - *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __20; - } - } - #pragma unroll - for (int i_24 = 0; i_24 < 2; ++i_24) { - for (int vec_8 = 0; vec_8 < 2; ++vec_8) { - uint2 __23; - float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__23))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__23))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]); - *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __23; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0); - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_25 = 0; i_25 < 2; ++i_25) { - *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104)); - for (int vec_9 = 0; vec_9 < 2; ++vec_9) { - float4 __24; - uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4)); - ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]); - ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]); - *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __24; - } - } - #pragma unroll - for (int i_26 = 0; i_26 < 2; ++i_26) { - *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056)); - for (int vec_10 = 0; vec_10 < 2; ++vec_10) { - float4 __25; - float4 __26; - float4 __27; - uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4)); - ((float2*)(&__27))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]); - ((float2*)(&__27))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]); - float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __26.x = (__27.x*v__24.x); - __26.y = (__27.y*v__24.y); - __26.z = (__27.z*v__24.z); - __26.w = (__27.w*v__24.w); - float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4))); - __25.x = (__26.x*v__25.x); - __25.y = (__26.y*v__25.y); - __25.z = (__26.z*v__25.z); - __25.w = (__26.w*v__25.w); - *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __25; - } - } - #pragma unroll - for (int i_27 = 0; i_27 < 2; ++i_27) { - for (int vec_11 = 0; vec_11 < 2; ++vec_11) { - uint2 __28; - float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__28))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__28))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]); - *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __28; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/25b1b2d19c0004c3d858fe560874c170531adddd2ba8ea1ea0a4020fd5aeb5a3/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/25b1b2d19c0004c3d858fe560874c170531adddd2ba8ea1ea0a4020fd5aeb5a3/host_kernel.cu deleted file mode 100644 index 66cdc86..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/25b1b2d19c0004c3d858fe560874c170531adddd2ba8ea1ea0a4020fd5aeb5a3/host_kernel.cu +++ /dev/null @@ -1,3359 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_pre_big_fuse_with_norm_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[16]; - void* stack_ffi_any = stack; - if (!((num_args == 9))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_pre_big_fuse_with_norm_tilelang: num_args should be 9", (long long)(num_args), (long long)(9)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang: args pointer is NULL"); - return -1; - } - int32_t gemm_out_mul_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((gemm_out_mul_type_index == 0) || (gemm_out_mul_type_index == 4)) || (gemm_out_mul_type_index == 7)) || (64 <= gemm_out_mul_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_mul expected pointer or tensor handle"); - return -1; - } - int32_t gemm_out_sqrsum_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((gemm_out_sqrsum_type_index == 0) || (gemm_out_sqrsum_type_index == 4)) || (gemm_out_sqrsum_type_index == 7)) || (64 <= gemm_out_sqrsum_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_sqrsum expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t layer_input_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((layer_input_type_index == 0) || (layer_input_type_index == 4)) || (layer_input_type_index == 7)) || (64 <= layer_input_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input layer_input expected pointer or tensor handle"); - return -1; - } - int32_t norm_weight_type_index = (((TVMFFIAny*)args)[8].type_index); - if (!(((((norm_weight_type_index == 0) || (norm_weight_type_index == 4)) || (norm_weight_type_index == 7)) || (64 <= norm_weight_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input norm_weight expected pointer or tensor handle"); - return -1; - } - void* gemm_out_mul = ((gemm_out_mul_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* gemm_out_sqrsum = ((gemm_out_sqrsum_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* layer_input = ((layer_input_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - void* norm_weight = ((norm_weight_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[8].v_ptr) + 24)) : (((TVMFFIAny*)args)[8].v_ptr)); - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null = (gemm_out_mul == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null = (gemm_out_sqrsum == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_residual_is_null = (residual == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null = (layer_input == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.layer_input is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null = (norm_weight == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.norm_weight is expected to have non-NULL pointer"); - return -1; - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape = (((DLTensor*)gemm_out_mul)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape = (((DLTensor*)gemm_out_sqrsum)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape = (((DLTensor*)layer_input)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape = (((DLTensor*)norm_weight)[0].shape); - if ((((DLTensor*)gemm_out_mul)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul, mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum, mhc_pre_big_fuse_with_norm_tilelang.residual, mhc_pre_big_fuse_with_norm_tilelang.post_mix, mhc_pre_big_fuse_with_norm_tilelang.comb_mix, mhc_pre_big_fuse_with_norm_tilelang.layer_input"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides = (((DLTensor*)gemm_out_mul)[0].strides); - int32_t dev_id = (((DLTensor*)gemm_out_mul)[0].device.device_id); - void* gemm_out_mul_1 = (((DLTensor*)gemm_out_mul)[0].data); - if ((((DLTensor*)gemm_out_sqrsum)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides = (((DLTensor*)gemm_out_sqrsum)[0].strides); - void* gemm_out_sqrsum_1 = (((DLTensor*)gemm_out_sqrsum)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)comb_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)layer_input)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides = (((DLTensor*)layer_input)[0].strides); - void* layer_input_1 = (((DLTensor*)layer_input)[0].data); - if ((((DLTensor*)norm_weight)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_9) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides = (((DLTensor*)norm_weight)[0].strides); - void* norm_weight_1 = (((DLTensor*)norm_weight)[0].data); - if ((((((DLTensor*)gemm_out_mul)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_mul)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_mul)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0]) != 26) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)26; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[1]); - } - if ((condval_2 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - int32_t condval_3; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[0]); - } - if (((num_tokens * 24) != condval_3) && (num_tokens != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_15) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_mul)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_mul)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_17) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_18) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)gemm_out_sqrsum)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0]) != 26) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)26; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[1]); - } - if ((condval_4 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_22) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[0]); - } - if ((((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != condval_5) && (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_23) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_sqrsum)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_26) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_27) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_28) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_29) != 0) { - return -1; - } - } - int32_t condval_6; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - if (condval_6 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_7; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_7); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_34) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_35) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_36) != 0) { - return -1; - } - } - int32_t condval_8; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - if (condval_8 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_9; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_9); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_38) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_40) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_41) != 0) { - return -1; - } - } - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_42) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_46) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - if (condval_12 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_13; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_13); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_47) != 0) { - return -1; - } - } - int32_t condval_14; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[0]); - } - if ((condval_14 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_48) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_49) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_51) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_52) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_53) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_54) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - if (condval_15 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_16; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_16); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_56) != 0) { - return -1; - } - } - int32_t condval_17; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[0]); - } - if ((condval_17 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_58) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_60) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_61) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_62) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_63) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1]) != 16) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_64) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - if (condval_18 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[0]); - } - if ((condval_20 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_66) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_67) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_68) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_69) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_70) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)layer_input)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)layer_input)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)layer_input)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_71) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_72) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_73) != 0) { - return -1; - } - } - int32_t condval_21; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - if (condval_21 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_22; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_22); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - int32_t condval_23; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[0]); - } - if ((condval_23 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_75) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)layer_input)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_76) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_77) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_78) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_79) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)norm_weight)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)norm_weight)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)norm_weight)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_80) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - if (condval_24 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_25; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_25); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)norm_weight)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_83) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_85) != 0) { - return -1; - } - } - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_86) != 0) { - return -1; - } - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_87) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = gemm_out_mul_1; - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = gemm_out_sqrsum_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = hc_scale_1; - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = layer_input_1; - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = norm_weight_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = post_mix_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)96); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)29040); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = (int64_t)0; - if (mhc_pre_big_fuse_with_norm_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_pre_big_fuse_with_norm_tilelang_kernel", &mhc_pre_big_fuse_with_norm_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_pre_big_fuse_with_norm_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 15, &result_88) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/25b1b2d19c0004c3d858fe560874c170531adddd2ba8ea1ea0a4020fd5aeb5a3/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/25b1b2d19c0004c3d858fe560874c170531adddd2ba8ea1ea0a4020fd5aeb5a3/params.pkl deleted file mode 100644 index 8d5b15c41f344395bcc23495e09928ea8ea8fdec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2200 zcmeHI&r2IY6t>31fUN{X`~d|!rwC!G3O(fPZK#NfmmQFa;H_QWn3w3xs@6jcJur!_gP5Sp&yFMXqciV>Mv#{ zK8Z$Cto-Y*SQ|OJDL#)(6JKV*t*O1D6+W%abp3g1JuKI1=i6khCL>y)jmH%zM -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float mixes[1]; - float rms[1]; - float cm[1]; - float row_max[1]; - float row_sum[1]; - float col_sum[1]; - float sumsq_per_pos[16]; - float sumsq[1]; - float rsqrt_norm[1]; - float xl[64]; - float ol[16]; - bfloat16_t xs_local_cast[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t xs_local_cast_2[8]; - float w_local[16]; - float ol_1[16]; - bfloat16_t w_shared_local_cast_3[8]; - bfloat16_t output_shared_local_cast_4[8]; - bfloat16_t layer_input_local_cast_5[8]; - bfloat16_t w_shared_local_cast_6[8]; - bfloat16_t output_shared_local_cast_7[8]; - bfloat16_t layer_input_local_cast_8[8]; - bfloat16_t w_shared_local_cast_9[8]; - bfloat16_t output_shared_local_cast_10[8]; - bfloat16_t layer_input_local_cast_11[8]; - mixes[0] = 0x0p+0f/*0.000000e+00*/; - rms[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - for (int i_split = 0; i_split < 13; ++i_split) { - rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]); - } - rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - mixes[0] = 0x0p+0f/*0.000000e+00*/; - for (int i_split_1 = 0; i_split_1 < 13; ++i_split_1) { - mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]); - } - mixes[0] = (mixes[0] * rms[0]); - if ((((int)threadIdx.x) / 24) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0]; - } - __syncthreads(); - if (((int)threadIdx.x) < 32) { - if (((int)threadIdx.x) < 4) { - post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/); - } - cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]); - row_max[0] = -CUDART_INF_F; - row_max[0] = max(row_max[0], cm[0]); - row_max[0] = tl::AllReduce>::run(row_max[0]); - cm[0] = expf((cm[0] - row_max[0])); - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - for (int __1 = 0; __1 < 19; ++__1) { - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - } - if ((((int)threadIdx.x) >> 4) == 0) { - comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0]; - } - } else { - if (((int)threadIdx.x) < 36) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - #pragma unroll - for (int i = 0; i < 4; ++i) { - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_2 = 0; i_2 < 8; ++i_2) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_4 = 0; i_4 < 8; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - tl::__sync_thread_partial<3, 64>(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)]; - #pragma unroll - for (int i_6 = 0; i_6 < 16; ++i_6) { - ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)])); - } - } - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - float4 __3; - float4 v__1 = *(float4*)(sumsq_per_pos + (i_7 * 4)); - float4 __4; - float4 v__2 = *(float4*)(ol + (i_7 * 4)); - __4.x = (v__2.x*v__2.x); - __4.y = (v__2.y*v__2.y); - __4.z = (v__2.z*v__2.z); - __4.w = (v__2.w*v__2.w); - __3.x = (v__1.x+__4.x); - __3.y = (v__1.y+__4.y); - __3.z = (v__1.z+__4.z); - __3.w = (v__1.w+__4.w); - *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3; - uint2 __5; - float4 v__3 = *(float4*)(ol + (i_7 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __5; - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_8 = 0; i_8 < 8; ++i_8) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __6; - uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]); - *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __6; - } - } - #pragma unroll - for (int i_9 = 0; i_9 < 4; ++i_9) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)]; - #pragma unroll - for (int i_10 = 0; i_10 < 16; ++i_10) { - ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)])); - } - } - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - float4 __7; - float4 v__5 = *(float4*)(sumsq_per_pos + (i_11 * 4)); - float4 __8; - float4 v__6 = *(float4*)(ol + (i_11 * 4)); - __8.x = (v__6.x*v__6.x); - __8.y = (v__6.y*v__6.y); - __8.z = (v__6.z*v__6.z); - __8.w = (v__6.w*v__6.w); - __7.x = (v__5.x+__8.x); - __7.y = (v__5.y+__8.y); - __7.z = (v__5.z+__8.z); - __7.w = (v__5.w+__8.w); - *(float4*)(sumsq_per_pos + (i_11 * 4)) = __7; - uint2 __9; - float4 v__7 = *(float4*)(ol + (i_11 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__9))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__9))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __9; - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_12 = 0; i_12 < 8; ++i_12) { - *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __10; - uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4)); - ((float2*)(&__10))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]); - ((float2*)(&__10))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]); - *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __10; - } - } - #pragma unroll - for (int i_13 = 0; i_13 < 4; ++i_13) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)]; - #pragma unroll - for (int i_14 = 0; i_14 < 16; ++i_14) { - ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)])); - } - } - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - float4 __11; - float4 v__9 = *(float4*)(sumsq_per_pos + (i_15 * 4)); - float4 __12; - float4 v__10 = *(float4*)(ol + (i_15 * 4)); - __12.x = (v__10.x*v__10.x); - __12.y = (v__10.y*v__10.y); - __12.z = (v__10.z*v__10.z); - __12.w = (v__10.w*v__10.w); - __11.x = (v__9.x+__12.x); - __11.y = (v__9.y+__12.y); - __11.z = (v__9.z+__12.z); - __11.w = (v__9.w+__12.w); - *(float4*)(sumsq_per_pos + (i_15 * 4)) = __11; - uint2 __13; - float4 v__11 = *(float4*)(ol + (i_15 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__13))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__13))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __13; - } - sumsq[0] = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int rv = 0; rv < 16; ++rv) { - sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]); - } - tl::__sync_thread_partial<3, 64>(); - sumsq[0] = tl::AllReduce>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192]))); - rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_17 = 0; i_17 < 2; ++i_17) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)]))); - } - tl::cp_async_commit(); - for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_18 = 0; i_18 < 2; ++i_18) { - *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_3 = 0; vec_3 < 2; ++vec_3) { - float4 __14; - uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4)); - ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]); - ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]); - *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __14; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_19 = 0; i_19 < 2; ++i_19) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)]))); - } - tl::cp_async_commit(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_20 = 0; i_20 < 2; ++i_20) { - *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984)); - for (int vec_4 = 0; vec_4 < 2; ++vec_4) { - float4 __15; - float4 __16; - float4 __17; - uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4)); - ((float2*)(&__17))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]); - ((float2*)(&__17))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]); - float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __16.x = (__17.x*v__14.x); - __16.y = (__17.y*v__14.y); - __16.z = (__17.z*v__14.z); - __16.w = (__17.w*v__14.w); - float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4))); - __15.x = (__16.x*v__15.x); - __15.y = (__16.y*v__15.y); - __15.z = (__16.z*v__15.z); - __15.w = (__16.w*v__15.w); - *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __15; - } - } - #pragma unroll - for (int i_21 = 0; i_21 < 2; ++i_21) { - for (int vec_5 = 0; vec_5 < 2; ++vec_5) { - uint2 __18; - float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__18))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__18))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]); - *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __18; - } - *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0); - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_22 = 0; i_22 < 2; ++i_22) { - *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_6 = 0; vec_6 < 2; ++vec_6) { - float4 __19; - uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4)); - ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]); - ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]); - *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __19; - } - } - #pragma unroll - for (int i_23 = 0; i_23 < 2; ++i_23) { - *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032)); - for (int vec_7 = 0; vec_7 < 2; ++vec_7) { - float4 __20; - float4 __21; - float4 __22; - uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4)); - ((float2*)(&__22))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]); - ((float2*)(&__22))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]); - float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __21.x = (__22.x*v__19.x); - __21.y = (__22.y*v__19.y); - __21.z = (__22.z*v__19.z); - __21.w = (__22.w*v__19.w); - float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4))); - __20.x = (__21.x*v__20.x); - __20.y = (__21.y*v__20.y); - __20.z = (__21.z*v__20.z); - __20.w = (__21.w*v__20.w); - *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __20; - } - } - #pragma unroll - for (int i_24 = 0; i_24 < 2; ++i_24) { - for (int vec_8 = 0; vec_8 < 2; ++vec_8) { - uint2 __23; - float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__23))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__23))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]); - *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __23; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0); - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_25 = 0; i_25 < 2; ++i_25) { - *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104)); - for (int vec_9 = 0; vec_9 < 2; ++vec_9) { - float4 __24; - uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4)); - ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]); - ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]); - *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __24; - } - } - #pragma unroll - for (int i_26 = 0; i_26 < 2; ++i_26) { - *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056)); - for (int vec_10 = 0; vec_10 < 2; ++vec_10) { - float4 __25; - float4 __26; - float4 __27; - uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4)); - ((float2*)(&__27))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]); - ((float2*)(&__27))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]); - float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __26.x = (__27.x*v__24.x); - __26.y = (__27.y*v__24.y); - __26.z = (__27.z*v__24.z); - __26.w = (__27.w*v__24.w); - float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4))); - __25.x = (__26.x*v__25.x); - __25.y = (__26.y*v__25.y); - __25.z = (__26.z*v__25.z); - __25.w = (__26.w*v__25.w); - *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __25; - } - } - #pragma unroll - for (int i_27 = 0; i_27 < 2; ++i_27) { - for (int vec_11 = 0; vec_11 < 2; ++vec_11) { - uint2 __28; - float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__28))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__28))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]); - *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __28; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2f65ba228083b031f9f8b7020d4a171fc67be94c7b476fa6fbe648831cff6878/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2f65ba228083b031f9f8b7020d4a171fc67be94c7b476fa6fbe648831cff6878/host_kernel.cu deleted file mode 100644 index 04f7c6f..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2f65ba228083b031f9f8b7020d4a171fc67be94c7b476fa6fbe648831cff6878/host_kernel.cu +++ /dev/null @@ -1,3359 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_pre_big_fuse_with_norm_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[16]; - void* stack_ffi_any = stack; - if (!((num_args == 9))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_pre_big_fuse_with_norm_tilelang: num_args should be 9", (long long)(num_args), (long long)(9)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang: args pointer is NULL"); - return -1; - } - int32_t gemm_out_mul_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((gemm_out_mul_type_index == 0) || (gemm_out_mul_type_index == 4)) || (gemm_out_mul_type_index == 7)) || (64 <= gemm_out_mul_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_mul expected pointer or tensor handle"); - return -1; - } - int32_t gemm_out_sqrsum_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((gemm_out_sqrsum_type_index == 0) || (gemm_out_sqrsum_type_index == 4)) || (gemm_out_sqrsum_type_index == 7)) || (64 <= gemm_out_sqrsum_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_sqrsum expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t layer_input_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((layer_input_type_index == 0) || (layer_input_type_index == 4)) || (layer_input_type_index == 7)) || (64 <= layer_input_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input layer_input expected pointer or tensor handle"); - return -1; - } - int32_t norm_weight_type_index = (((TVMFFIAny*)args)[8].type_index); - if (!(((((norm_weight_type_index == 0) || (norm_weight_type_index == 4)) || (norm_weight_type_index == 7)) || (64 <= norm_weight_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input norm_weight expected pointer or tensor handle"); - return -1; - } - void* gemm_out_mul = ((gemm_out_mul_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* gemm_out_sqrsum = ((gemm_out_sqrsum_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* layer_input = ((layer_input_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - void* norm_weight = ((norm_weight_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[8].v_ptr) + 24)) : (((TVMFFIAny*)args)[8].v_ptr)); - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null = (gemm_out_mul == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null = (gemm_out_sqrsum == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_residual_is_null = (residual == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null = (layer_input == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.layer_input is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null = (norm_weight == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.norm_weight is expected to have non-NULL pointer"); - return -1; - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape = (((DLTensor*)gemm_out_mul)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape = (((DLTensor*)gemm_out_sqrsum)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape = (((DLTensor*)layer_input)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape = (((DLTensor*)norm_weight)[0].shape); - if ((((DLTensor*)gemm_out_mul)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul, mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum, mhc_pre_big_fuse_with_norm_tilelang.residual, mhc_pre_big_fuse_with_norm_tilelang.post_mix, mhc_pre_big_fuse_with_norm_tilelang.comb_mix, mhc_pre_big_fuse_with_norm_tilelang.layer_input"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides = (((DLTensor*)gemm_out_mul)[0].strides); - int32_t dev_id = (((DLTensor*)gemm_out_mul)[0].device.device_id); - void* gemm_out_mul_1 = (((DLTensor*)gemm_out_mul)[0].data); - if ((((DLTensor*)gemm_out_sqrsum)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides = (((DLTensor*)gemm_out_sqrsum)[0].strides); - void* gemm_out_sqrsum_1 = (((DLTensor*)gemm_out_sqrsum)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)comb_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)layer_input)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides = (((DLTensor*)layer_input)[0].strides); - void* layer_input_1 = (((DLTensor*)layer_input)[0].data); - if ((((DLTensor*)norm_weight)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_9) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides = (((DLTensor*)norm_weight)[0].strides); - void* norm_weight_1 = (((DLTensor*)norm_weight)[0].data); - if ((((((DLTensor*)gemm_out_mul)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_mul)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_mul)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0]) != 13) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)13; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[1]); - } - if ((condval_2 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - int32_t condval_3; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[0]); - } - if (((num_tokens * 24) != condval_3) && (num_tokens != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_15) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_mul)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_mul)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_17) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_18) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)gemm_out_sqrsum)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0]) != 13) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)13; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[1]); - } - if ((condval_4 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_22) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[0]); - } - if ((((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != condval_5) && (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_23) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_sqrsum)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_26) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_27) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_28) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_29) != 0) { - return -1; - } - } - int32_t condval_6; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - if (condval_6 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_7; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_7); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_34) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_35) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_36) != 0) { - return -1; - } - } - int32_t condval_8; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - if (condval_8 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_9; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_9); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_38) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_40) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_41) != 0) { - return -1; - } - } - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_42) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_46) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - if (condval_12 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_13; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_13); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_47) != 0) { - return -1; - } - } - int32_t condval_14; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[0]); - } - if ((condval_14 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_48) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_49) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_51) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_52) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_53) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_54) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - if (condval_15 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_16; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_16); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_56) != 0) { - return -1; - } - } - int32_t condval_17; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[0]); - } - if ((condval_17 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_58) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_60) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_61) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_62) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_63) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1]) != 16) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_64) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - if (condval_18 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[0]); - } - if ((condval_20 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_66) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_67) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_68) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_69) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_70) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)layer_input)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)layer_input)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)layer_input)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_71) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_72) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_73) != 0) { - return -1; - } - } - int32_t condval_21; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - if (condval_21 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_22; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_22); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - int32_t condval_23; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[0]); - } - if ((condval_23 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_75) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)layer_input)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_76) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_77) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_78) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_79) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)norm_weight)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)norm_weight)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)norm_weight)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_80) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - if (condval_24 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_25; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_25); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)norm_weight)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_83) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_85) != 0) { - return -1; - } - } - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_86) != 0) { - return -1; - } - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_87) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = gemm_out_mul_1; - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = gemm_out_sqrsum_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = hc_scale_1; - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = layer_input_1; - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = norm_weight_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = post_mix_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)96); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)29040); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = (int64_t)0; - if (mhc_pre_big_fuse_with_norm_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_pre_big_fuse_with_norm_tilelang_kernel", &mhc_pre_big_fuse_with_norm_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_pre_big_fuse_with_norm_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 15, &result_88) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2f65ba228083b031f9f8b7020d4a171fc67be94c7b476fa6fbe648831cff6878/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/2f65ba228083b031f9f8b7020d4a171fc67be94c7b476fa6fbe648831cff6878/params.pkl deleted file mode 100644 index da26865775ab97650f6b8125928324b32e83500d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2200 zcmeHI&r2IY6t>2s0e=t>sR}}!Q-rWo1rIs*HdMrl7m;B$`;r;EJG0DA)Djlz!Jek4 zjDMg1O5bi0wJKC{jeDEN_ulut@9mrURlWODpDVv}-*jtM#t?JW3qbXxf*|FFC*Cbm zFbd+M@-*~2H$F#nHL^cb@NUIkCal|)0UHHGZRp*{ZmhZ8-}ATr#vkU0Vq@4C<(4LC z9c{462J0B$7FegO!n#H$%=Xh5{1C~Wq3d6{a+677QWprOvWn@g1_ns_8R~I diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/34141341ae4687b7d5cb99f48bcdb971fae6efe8bba5eed695620f0e86b8bdd4/device_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/34141341ae4687b7d5cb99f48bcdb971fae6efe8bba5eed695620f0e86b8bdd4/device_kernel.cu deleted file mode 100644 index 4e7319d..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/34141341ae4687b7d5cb99f48bcdb971fae6efe8bba5eed695620f0e86b8bdd4/device_kernel.cu +++ /dev/null @@ -1,424 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float mixes[1]; - float rms[1]; - float cm[1]; - float row_max[1]; - float row_sum[1]; - float col_sum[1]; - float sumsq_per_pos[16]; - float sumsq[1]; - float rsqrt_norm[1]; - float xl[64]; - float ol[16]; - bfloat16_t xs_local_cast[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t xs_local_cast_2[8]; - float w_local[16]; - float ol_1[16]; - bfloat16_t w_shared_local_cast_3[8]; - bfloat16_t output_shared_local_cast_4[8]; - bfloat16_t layer_input_local_cast_5[8]; - bfloat16_t w_shared_local_cast_6[8]; - bfloat16_t output_shared_local_cast_7[8]; - bfloat16_t layer_input_local_cast_8[8]; - bfloat16_t w_shared_local_cast_9[8]; - bfloat16_t output_shared_local_cast_10[8]; - bfloat16_t layer_input_local_cast_11[8]; - mixes[0] = 0x0p+0f/*0.000000e+00*/; - rms[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - for (int i_split = 0; i_split < 39; ++i_split) { - rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]); - } - rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - mixes[0] = 0x0p+0f/*0.000000e+00*/; - for (int i_split_1 = 0; i_split_1 < 39; ++i_split_1) { - mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]); - } - mixes[0] = (mixes[0] * rms[0]); - if ((((int)threadIdx.x) / 24) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0]; - } - __syncthreads(); - if (((int)threadIdx.x) < 32) { - if (((int)threadIdx.x) < 4) { - post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/); - } - cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]); - row_max[0] = -CUDART_INF_F; - row_max[0] = max(row_max[0], cm[0]); - row_max[0] = tl::AllReduce>::run(row_max[0]); - cm[0] = expf((cm[0] - row_max[0])); - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - for (int __1 = 0; __1 < 19; ++__1) { - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - } - if ((((int)threadIdx.x) >> 4) == 0) { - comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0]; - } - } else { - if (((int)threadIdx.x) < 36) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - #pragma unroll - for (int i = 0; i < 4; ++i) { - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_2 = 0; i_2 < 8; ++i_2) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_4 = 0; i_4 < 8; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - tl::__sync_thread_partial<3, 64>(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)]; - #pragma unroll - for (int i_6 = 0; i_6 < 16; ++i_6) { - ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)])); - } - } - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - float4 __3; - float4 v__1 = *(float4*)(sumsq_per_pos + (i_7 * 4)); - float4 __4; - float4 v__2 = *(float4*)(ol + (i_7 * 4)); - __4.x = (v__2.x*v__2.x); - __4.y = (v__2.y*v__2.y); - __4.z = (v__2.z*v__2.z); - __4.w = (v__2.w*v__2.w); - __3.x = (v__1.x+__4.x); - __3.y = (v__1.y+__4.y); - __3.z = (v__1.z+__4.z); - __3.w = (v__1.w+__4.w); - *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3; - uint2 __5; - float4 v__3 = *(float4*)(ol + (i_7 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __5; - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_8 = 0; i_8 < 8; ++i_8) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __6; - uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]); - *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __6; - } - } - #pragma unroll - for (int i_9 = 0; i_9 < 4; ++i_9) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)]; - #pragma unroll - for (int i_10 = 0; i_10 < 16; ++i_10) { - ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)])); - } - } - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - float4 __7; - float4 v__5 = *(float4*)(sumsq_per_pos + (i_11 * 4)); - float4 __8; - float4 v__6 = *(float4*)(ol + (i_11 * 4)); - __8.x = (v__6.x*v__6.x); - __8.y = (v__6.y*v__6.y); - __8.z = (v__6.z*v__6.z); - __8.w = (v__6.w*v__6.w); - __7.x = (v__5.x+__8.x); - __7.y = (v__5.y+__8.y); - __7.z = (v__5.z+__8.z); - __7.w = (v__5.w+__8.w); - *(float4*)(sumsq_per_pos + (i_11 * 4)) = __7; - uint2 __9; - float4 v__7 = *(float4*)(ol + (i_11 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__9))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__9))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __9; - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_12 = 0; i_12 < 8; ++i_12) { - *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __10; - uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4)); - ((float2*)(&__10))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]); - ((float2*)(&__10))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]); - *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __10; - } - } - #pragma unroll - for (int i_13 = 0; i_13 < 4; ++i_13) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)]; - #pragma unroll - for (int i_14 = 0; i_14 < 16; ++i_14) { - ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)])); - } - } - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - float4 __11; - float4 v__9 = *(float4*)(sumsq_per_pos + (i_15 * 4)); - float4 __12; - float4 v__10 = *(float4*)(ol + (i_15 * 4)); - __12.x = (v__10.x*v__10.x); - __12.y = (v__10.y*v__10.y); - __12.z = (v__10.z*v__10.z); - __12.w = (v__10.w*v__10.w); - __11.x = (v__9.x+__12.x); - __11.y = (v__9.y+__12.y); - __11.z = (v__9.z+__12.z); - __11.w = (v__9.w+__12.w); - *(float4*)(sumsq_per_pos + (i_15 * 4)) = __11; - uint2 __13; - float4 v__11 = *(float4*)(ol + (i_15 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__13))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__13))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __13; - } - sumsq[0] = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int rv = 0; rv < 16; ++rv) { - sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]); - } - tl::__sync_thread_partial<3, 64>(); - sumsq[0] = tl::AllReduce>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192]))); - rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_17 = 0; i_17 < 2; ++i_17) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)]))); - } - tl::cp_async_commit(); - for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_18 = 0; i_18 < 2; ++i_18) { - *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_3 = 0; vec_3 < 2; ++vec_3) { - float4 __14; - uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4)); - ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]); - ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]); - *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __14; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_19 = 0; i_19 < 2; ++i_19) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)]))); - } - tl::cp_async_commit(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_20 = 0; i_20 < 2; ++i_20) { - *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984)); - for (int vec_4 = 0; vec_4 < 2; ++vec_4) { - float4 __15; - float4 __16; - float4 __17; - uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4)); - ((float2*)(&__17))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]); - ((float2*)(&__17))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]); - float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __16.x = (__17.x*v__14.x); - __16.y = (__17.y*v__14.y); - __16.z = (__17.z*v__14.z); - __16.w = (__17.w*v__14.w); - float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4))); - __15.x = (__16.x*v__15.x); - __15.y = (__16.y*v__15.y); - __15.z = (__16.z*v__15.z); - __15.w = (__16.w*v__15.w); - *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __15; - } - } - #pragma unroll - for (int i_21 = 0; i_21 < 2; ++i_21) { - for (int vec_5 = 0; vec_5 < 2; ++vec_5) { - uint2 __18; - float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__18))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__18))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]); - *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __18; - } - *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0); - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_22 = 0; i_22 < 2; ++i_22) { - *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_6 = 0; vec_6 < 2; ++vec_6) { - float4 __19; - uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4)); - ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]); - ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]); - *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __19; - } - } - #pragma unroll - for (int i_23 = 0; i_23 < 2; ++i_23) { - *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032)); - for (int vec_7 = 0; vec_7 < 2; ++vec_7) { - float4 __20; - float4 __21; - float4 __22; - uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4)); - ((float2*)(&__22))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]); - ((float2*)(&__22))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]); - float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __21.x = (__22.x*v__19.x); - __21.y = (__22.y*v__19.y); - __21.z = (__22.z*v__19.z); - __21.w = (__22.w*v__19.w); - float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4))); - __20.x = (__21.x*v__20.x); - __20.y = (__21.y*v__20.y); - __20.z = (__21.z*v__20.z); - __20.w = (__21.w*v__20.w); - *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __20; - } - } - #pragma unroll - for (int i_24 = 0; i_24 < 2; ++i_24) { - for (int vec_8 = 0; vec_8 < 2; ++vec_8) { - uint2 __23; - float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__23))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__23))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]); - *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __23; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0); - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_25 = 0; i_25 < 2; ++i_25) { - *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104)); - for (int vec_9 = 0; vec_9 < 2; ++vec_9) { - float4 __24; - uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4)); - ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]); - ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]); - *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __24; - } - } - #pragma unroll - for (int i_26 = 0; i_26 < 2; ++i_26) { - *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056)); - for (int vec_10 = 0; vec_10 < 2; ++vec_10) { - float4 __25; - float4 __26; - float4 __27; - uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4)); - ((float2*)(&__27))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]); - ((float2*)(&__27))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]); - float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __26.x = (__27.x*v__24.x); - __26.y = (__27.y*v__24.y); - __26.z = (__27.z*v__24.z); - __26.w = (__27.w*v__24.w); - float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4))); - __25.x = (__26.x*v__25.x); - __25.y = (__26.y*v__25.y); - __25.z = (__26.z*v__25.z); - __25.w = (__26.w*v__25.w); - *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __25; - } - } - #pragma unroll - for (int i_27 = 0; i_27 < 2; ++i_27) { - for (int vec_11 = 0; vec_11 < 2; ++vec_11) { - uint2 __28; - float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__28))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__28))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]); - *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __28; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/34141341ae4687b7d5cb99f48bcdb971fae6efe8bba5eed695620f0e86b8bdd4/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/34141341ae4687b7d5cb99f48bcdb971fae6efe8bba5eed695620f0e86b8bdd4/host_kernel.cu deleted file mode 100644 index 0b74380..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/34141341ae4687b7d5cb99f48bcdb971fae6efe8bba5eed695620f0e86b8bdd4/host_kernel.cu +++ /dev/null @@ -1,3359 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_pre_big_fuse_with_norm_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[16]; - void* stack_ffi_any = stack; - if (!((num_args == 9))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_pre_big_fuse_with_norm_tilelang: num_args should be 9", (long long)(num_args), (long long)(9)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang: args pointer is NULL"); - return -1; - } - int32_t gemm_out_mul_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((gemm_out_mul_type_index == 0) || (gemm_out_mul_type_index == 4)) || (gemm_out_mul_type_index == 7)) || (64 <= gemm_out_mul_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_mul expected pointer or tensor handle"); - return -1; - } - int32_t gemm_out_sqrsum_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((gemm_out_sqrsum_type_index == 0) || (gemm_out_sqrsum_type_index == 4)) || (gemm_out_sqrsum_type_index == 7)) || (64 <= gemm_out_sqrsum_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_sqrsum expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t layer_input_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((layer_input_type_index == 0) || (layer_input_type_index == 4)) || (layer_input_type_index == 7)) || (64 <= layer_input_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input layer_input expected pointer or tensor handle"); - return -1; - } - int32_t norm_weight_type_index = (((TVMFFIAny*)args)[8].type_index); - if (!(((((norm_weight_type_index == 0) || (norm_weight_type_index == 4)) || (norm_weight_type_index == 7)) || (64 <= norm_weight_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input norm_weight expected pointer or tensor handle"); - return -1; - } - void* gemm_out_mul = ((gemm_out_mul_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* gemm_out_sqrsum = ((gemm_out_sqrsum_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* layer_input = ((layer_input_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - void* norm_weight = ((norm_weight_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[8].v_ptr) + 24)) : (((TVMFFIAny*)args)[8].v_ptr)); - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null = (gemm_out_mul == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null = (gemm_out_sqrsum == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_residual_is_null = (residual == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null = (layer_input == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.layer_input is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null = (norm_weight == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.norm_weight is expected to have non-NULL pointer"); - return -1; - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape = (((DLTensor*)gemm_out_mul)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape = (((DLTensor*)gemm_out_sqrsum)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape = (((DLTensor*)layer_input)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape = (((DLTensor*)norm_weight)[0].shape); - if ((((DLTensor*)gemm_out_mul)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul, mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum, mhc_pre_big_fuse_with_norm_tilelang.residual, mhc_pre_big_fuse_with_norm_tilelang.post_mix, mhc_pre_big_fuse_with_norm_tilelang.comb_mix, mhc_pre_big_fuse_with_norm_tilelang.layer_input"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides = (((DLTensor*)gemm_out_mul)[0].strides); - int32_t dev_id = (((DLTensor*)gemm_out_mul)[0].device.device_id); - void* gemm_out_mul_1 = (((DLTensor*)gemm_out_mul)[0].data); - if ((((DLTensor*)gemm_out_sqrsum)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides = (((DLTensor*)gemm_out_sqrsum)[0].strides); - void* gemm_out_sqrsum_1 = (((DLTensor*)gemm_out_sqrsum)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)comb_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)layer_input)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides = (((DLTensor*)layer_input)[0].strides); - void* layer_input_1 = (((DLTensor*)layer_input)[0].data); - if ((((DLTensor*)norm_weight)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_9) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides = (((DLTensor*)norm_weight)[0].strides); - void* norm_weight_1 = (((DLTensor*)norm_weight)[0].data); - if ((((((DLTensor*)gemm_out_mul)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_mul)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_mul)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0]) != 39) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)39; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[1]); - } - if ((condval_2 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - int32_t condval_3; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[0]); - } - if (((num_tokens * 24) != condval_3) && (num_tokens != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_15) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_mul)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_mul)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_17) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_18) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)gemm_out_sqrsum)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0]) != 39) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)39; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[1]); - } - if ((condval_4 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_22) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[0]); - } - if ((((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != condval_5) && (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_23) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_sqrsum)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_26) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_27) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_28) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_29) != 0) { - return -1; - } - } - int32_t condval_6; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - if (condval_6 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_7; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_7); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_34) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_35) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_36) != 0) { - return -1; - } - } - int32_t condval_8; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - if (condval_8 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_9; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_9); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_38) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_40) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_41) != 0) { - return -1; - } - } - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_42) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_46) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - if (condval_12 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_13; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_13); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_47) != 0) { - return -1; - } - } - int32_t condval_14; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[0]); - } - if ((condval_14 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_48) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_49) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_51) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_52) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_53) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_54) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - if (condval_15 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_16; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_16); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_56) != 0) { - return -1; - } - } - int32_t condval_17; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[0]); - } - if ((condval_17 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_58) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_60) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_61) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_62) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_63) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1]) != 16) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_64) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - if (condval_18 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[0]); - } - if ((condval_20 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_66) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_67) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_68) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_69) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_70) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)layer_input)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)layer_input)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)layer_input)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_71) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_72) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_73) != 0) { - return -1; - } - } - int32_t condval_21; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - if (condval_21 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_22; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_22); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - int32_t condval_23; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[0]); - } - if ((condval_23 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_75) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)layer_input)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_76) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_77) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_78) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_79) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)norm_weight)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)norm_weight)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)norm_weight)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_80) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - if (condval_24 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_25; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_25); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)norm_weight)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_83) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_85) != 0) { - return -1; - } - } - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_86) != 0) { - return -1; - } - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_87) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = gemm_out_mul_1; - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = gemm_out_sqrsum_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = hc_scale_1; - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = layer_input_1; - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = norm_weight_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = post_mix_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)96); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)29040); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = (int64_t)0; - if (mhc_pre_big_fuse_with_norm_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_pre_big_fuse_with_norm_tilelang_kernel", &mhc_pre_big_fuse_with_norm_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_pre_big_fuse_with_norm_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 15, &result_88) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/34141341ae4687b7d5cb99f48bcdb971fae6efe8bba5eed695620f0e86b8bdd4/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/34141341ae4687b7d5cb99f48bcdb971fae6efe8bba5eed695620f0e86b8bdd4/params.pkl deleted file mode 100644 index d01ecff818bf4ba28e62622fd646dfe6da90c92f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2200 zcmeHIK}#D!6t<0t0b2T zU2U++I_nzX7Ff5e!g@w0%ns5R{0PY&q3iqH^6v%x$kVrUlfJS3tIvM>&{Z?RF^>Jv zFJi?lGX)n+ho{iB2rdf+R+<1VQ{!Fb2R9hL#%vT>6I?~e=>80e(b_VpBDf@--4;;qmRqTjp`GU^xz9qn27^#k#={gfQGYTs z@m@5V;={lGijQMwH^is0Y2x!dxHYp^w8Ce#nXWxft%v1W?d(n0YC57t+W5Ny<%q -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float mixes[1]; - float rms[1]; - float cm[1]; - float row_max[1]; - float row_sum[1]; - float col_sum[1]; - float sumsq_per_pos[16]; - float sumsq[1]; - float rsqrt_norm[1]; - float xl[64]; - float ol[16]; - bfloat16_t xs_local_cast[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t xs_local_cast_2[8]; - float w_local[16]; - float ol_1[16]; - bfloat16_t w_shared_local_cast_3[8]; - bfloat16_t output_shared_local_cast_4[8]; - bfloat16_t layer_input_local_cast_5[8]; - bfloat16_t w_shared_local_cast_6[8]; - bfloat16_t output_shared_local_cast_7[8]; - bfloat16_t layer_input_local_cast_8[8]; - bfloat16_t w_shared_local_cast_9[8]; - bfloat16_t output_shared_local_cast_10[8]; - bfloat16_t layer_input_local_cast_11[8]; - mixes[0] = 0x0p+0f/*0.000000e+00*/; - rms[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - for (int i_split = 0; i_split < 8; ++i_split) { - rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]); - } - rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - mixes[0] = 0x0p+0f/*0.000000e+00*/; - for (int i_split_1 = 0; i_split_1 < 8; ++i_split_1) { - mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]); - } - mixes[0] = (mixes[0] * rms[0]); - if ((((int)threadIdx.x) / 24) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0]; - } - __syncthreads(); - if (((int)threadIdx.x) < 32) { - if (((int)threadIdx.x) < 4) { - post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/); - } - cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]); - row_max[0] = -CUDART_INF_F; - row_max[0] = max(row_max[0], cm[0]); - row_max[0] = tl::AllReduce>::run(row_max[0]); - cm[0] = expf((cm[0] - row_max[0])); - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - for (int __1 = 0; __1 < 19; ++__1) { - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - } - if ((((int)threadIdx.x) >> 4) == 0) { - comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0]; - } - } else { - if (((int)threadIdx.x) < 36) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - #pragma unroll - for (int i = 0; i < 4; ++i) { - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_2 = 0; i_2 < 8; ++i_2) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_4 = 0; i_4 < 8; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - tl::__sync_thread_partial<3, 64>(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)]; - #pragma unroll - for (int i_6 = 0; i_6 < 16; ++i_6) { - ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)])); - } - } - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - float4 __3; - float4 v__1 = *(float4*)(sumsq_per_pos + (i_7 * 4)); - float4 __4; - float4 v__2 = *(float4*)(ol + (i_7 * 4)); - __4.x = (v__2.x*v__2.x); - __4.y = (v__2.y*v__2.y); - __4.z = (v__2.z*v__2.z); - __4.w = (v__2.w*v__2.w); - __3.x = (v__1.x+__4.x); - __3.y = (v__1.y+__4.y); - __3.z = (v__1.z+__4.z); - __3.w = (v__1.w+__4.w); - *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3; - uint2 __5; - float4 v__3 = *(float4*)(ol + (i_7 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __5; - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_8 = 0; i_8 < 8; ++i_8) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __6; - uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]); - *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __6; - } - } - #pragma unroll - for (int i_9 = 0; i_9 < 4; ++i_9) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)]; - #pragma unroll - for (int i_10 = 0; i_10 < 16; ++i_10) { - ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)])); - } - } - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - float4 __7; - float4 v__5 = *(float4*)(sumsq_per_pos + (i_11 * 4)); - float4 __8; - float4 v__6 = *(float4*)(ol + (i_11 * 4)); - __8.x = (v__6.x*v__6.x); - __8.y = (v__6.y*v__6.y); - __8.z = (v__6.z*v__6.z); - __8.w = (v__6.w*v__6.w); - __7.x = (v__5.x+__8.x); - __7.y = (v__5.y+__8.y); - __7.z = (v__5.z+__8.z); - __7.w = (v__5.w+__8.w); - *(float4*)(sumsq_per_pos + (i_11 * 4)) = __7; - uint2 __9; - float4 v__7 = *(float4*)(ol + (i_11 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__9))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__9))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __9; - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_12 = 0; i_12 < 8; ++i_12) { - *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __10; - uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4)); - ((float2*)(&__10))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]); - ((float2*)(&__10))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]); - *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __10; - } - } - #pragma unroll - for (int i_13 = 0; i_13 < 4; ++i_13) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)]; - #pragma unroll - for (int i_14 = 0; i_14 < 16; ++i_14) { - ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)])); - } - } - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - float4 __11; - float4 v__9 = *(float4*)(sumsq_per_pos + (i_15 * 4)); - float4 __12; - float4 v__10 = *(float4*)(ol + (i_15 * 4)); - __12.x = (v__10.x*v__10.x); - __12.y = (v__10.y*v__10.y); - __12.z = (v__10.z*v__10.z); - __12.w = (v__10.w*v__10.w); - __11.x = (v__9.x+__12.x); - __11.y = (v__9.y+__12.y); - __11.z = (v__9.z+__12.z); - __11.w = (v__9.w+__12.w); - *(float4*)(sumsq_per_pos + (i_15 * 4)) = __11; - uint2 __13; - float4 v__11 = *(float4*)(ol + (i_15 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__13))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__13))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __13; - } - sumsq[0] = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int rv = 0; rv < 16; ++rv) { - sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]); - } - tl::__sync_thread_partial<3, 64>(); - sumsq[0] = tl::AllReduce>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192]))); - rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_17 = 0; i_17 < 2; ++i_17) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)]))); - } - tl::cp_async_commit(); - for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_18 = 0; i_18 < 2; ++i_18) { - *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_3 = 0; vec_3 < 2; ++vec_3) { - float4 __14; - uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4)); - ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]); - ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]); - *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __14; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_19 = 0; i_19 < 2; ++i_19) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)]))); - } - tl::cp_async_commit(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_20 = 0; i_20 < 2; ++i_20) { - *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984)); - for (int vec_4 = 0; vec_4 < 2; ++vec_4) { - float4 __15; - float4 __16; - float4 __17; - uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4)); - ((float2*)(&__17))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]); - ((float2*)(&__17))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]); - float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __16.x = (__17.x*v__14.x); - __16.y = (__17.y*v__14.y); - __16.z = (__17.z*v__14.z); - __16.w = (__17.w*v__14.w); - float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4))); - __15.x = (__16.x*v__15.x); - __15.y = (__16.y*v__15.y); - __15.z = (__16.z*v__15.z); - __15.w = (__16.w*v__15.w); - *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __15; - } - } - #pragma unroll - for (int i_21 = 0; i_21 < 2; ++i_21) { - for (int vec_5 = 0; vec_5 < 2; ++vec_5) { - uint2 __18; - float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__18))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__18))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]); - *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __18; - } - *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0); - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_22 = 0; i_22 < 2; ++i_22) { - *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_6 = 0; vec_6 < 2; ++vec_6) { - float4 __19; - uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4)); - ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]); - ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]); - *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __19; - } - } - #pragma unroll - for (int i_23 = 0; i_23 < 2; ++i_23) { - *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032)); - for (int vec_7 = 0; vec_7 < 2; ++vec_7) { - float4 __20; - float4 __21; - float4 __22; - uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4)); - ((float2*)(&__22))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]); - ((float2*)(&__22))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]); - float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __21.x = (__22.x*v__19.x); - __21.y = (__22.y*v__19.y); - __21.z = (__22.z*v__19.z); - __21.w = (__22.w*v__19.w); - float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4))); - __20.x = (__21.x*v__20.x); - __20.y = (__21.y*v__20.y); - __20.z = (__21.z*v__20.z); - __20.w = (__21.w*v__20.w); - *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __20; - } - } - #pragma unroll - for (int i_24 = 0; i_24 < 2; ++i_24) { - for (int vec_8 = 0; vec_8 < 2; ++vec_8) { - uint2 __23; - float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__23))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__23))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]); - *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __23; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0); - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_25 = 0; i_25 < 2; ++i_25) { - *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104)); - for (int vec_9 = 0; vec_9 < 2; ++vec_9) { - float4 __24; - uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4)); - ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]); - ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]); - *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __24; - } - } - #pragma unroll - for (int i_26 = 0; i_26 < 2; ++i_26) { - *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056)); - for (int vec_10 = 0; vec_10 < 2; ++vec_10) { - float4 __25; - float4 __26; - float4 __27; - uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4)); - ((float2*)(&__27))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]); - ((float2*)(&__27))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]); - float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __26.x = (__27.x*v__24.x); - __26.y = (__27.y*v__24.y); - __26.z = (__27.z*v__24.z); - __26.w = (__27.w*v__24.w); - float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4))); - __25.x = (__26.x*v__25.x); - __25.y = (__26.y*v__25.y); - __25.z = (__26.z*v__25.z); - __25.w = (__26.w*v__25.w); - *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __25; - } - } - #pragma unroll - for (int i_27 = 0; i_27 < 2; ++i_27) { - for (int vec_11 = 0; vec_11 < 2; ++vec_11) { - uint2 __28; - float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__28))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__28))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]); - *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __28; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/455ef1b63b3a592f4741c938fcfb8d01f90e8269880307cffbf4438ed668de0b/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/455ef1b63b3a592f4741c938fcfb8d01f90e8269880307cffbf4438ed668de0b/host_kernel.cu deleted file mode 100644 index ac65b4a..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/455ef1b63b3a592f4741c938fcfb8d01f90e8269880307cffbf4438ed668de0b/host_kernel.cu +++ /dev/null @@ -1,3359 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_pre_big_fuse_with_norm_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[16]; - void* stack_ffi_any = stack; - if (!((num_args == 9))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_pre_big_fuse_with_norm_tilelang: num_args should be 9", (long long)(num_args), (long long)(9)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang: args pointer is NULL"); - return -1; - } - int32_t gemm_out_mul_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((gemm_out_mul_type_index == 0) || (gemm_out_mul_type_index == 4)) || (gemm_out_mul_type_index == 7)) || (64 <= gemm_out_mul_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_mul expected pointer or tensor handle"); - return -1; - } - int32_t gemm_out_sqrsum_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((gemm_out_sqrsum_type_index == 0) || (gemm_out_sqrsum_type_index == 4)) || (gemm_out_sqrsum_type_index == 7)) || (64 <= gemm_out_sqrsum_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_sqrsum expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t layer_input_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((layer_input_type_index == 0) || (layer_input_type_index == 4)) || (layer_input_type_index == 7)) || (64 <= layer_input_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input layer_input expected pointer or tensor handle"); - return -1; - } - int32_t norm_weight_type_index = (((TVMFFIAny*)args)[8].type_index); - if (!(((((norm_weight_type_index == 0) || (norm_weight_type_index == 4)) || (norm_weight_type_index == 7)) || (64 <= norm_weight_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input norm_weight expected pointer or tensor handle"); - return -1; - } - void* gemm_out_mul = ((gemm_out_mul_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* gemm_out_sqrsum = ((gemm_out_sqrsum_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* layer_input = ((layer_input_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - void* norm_weight = ((norm_weight_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[8].v_ptr) + 24)) : (((TVMFFIAny*)args)[8].v_ptr)); - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null = (gemm_out_mul == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null = (gemm_out_sqrsum == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_residual_is_null = (residual == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null = (layer_input == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.layer_input is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null = (norm_weight == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.norm_weight is expected to have non-NULL pointer"); - return -1; - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape = (((DLTensor*)gemm_out_mul)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape = (((DLTensor*)gemm_out_sqrsum)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape = (((DLTensor*)layer_input)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape = (((DLTensor*)norm_weight)[0].shape); - if ((((DLTensor*)gemm_out_mul)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul, mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum, mhc_pre_big_fuse_with_norm_tilelang.residual, mhc_pre_big_fuse_with_norm_tilelang.post_mix, mhc_pre_big_fuse_with_norm_tilelang.comb_mix, mhc_pre_big_fuse_with_norm_tilelang.layer_input"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides = (((DLTensor*)gemm_out_mul)[0].strides); - int32_t dev_id = (((DLTensor*)gemm_out_mul)[0].device.device_id); - void* gemm_out_mul_1 = (((DLTensor*)gemm_out_mul)[0].data); - if ((((DLTensor*)gemm_out_sqrsum)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides = (((DLTensor*)gemm_out_sqrsum)[0].strides); - void* gemm_out_sqrsum_1 = (((DLTensor*)gemm_out_sqrsum)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)comb_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)layer_input)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides = (((DLTensor*)layer_input)[0].strides); - void* layer_input_1 = (((DLTensor*)layer_input)[0].data); - if ((((DLTensor*)norm_weight)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_9) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides = (((DLTensor*)norm_weight)[0].strides); - void* norm_weight_1 = (((DLTensor*)norm_weight)[0].data); - if ((((((DLTensor*)gemm_out_mul)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_mul)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_mul)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0]) != 8) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)8; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[1]); - } - if ((condval_2 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - int32_t condval_3; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[0]); - } - if (((num_tokens * 24) != condval_3) && (num_tokens != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_15) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_mul)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_mul)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_17) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_18) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)gemm_out_sqrsum)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0]) != 8) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)8; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[1]); - } - if ((condval_4 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_22) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[0]); - } - if ((((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != condval_5) && (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_23) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_sqrsum)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_26) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_27) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_28) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_29) != 0) { - return -1; - } - } - int32_t condval_6; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - if (condval_6 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_7; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_7); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_34) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_35) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_36) != 0) { - return -1; - } - } - int32_t condval_8; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - if (condval_8 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_9; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_9); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_38) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_40) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_41) != 0) { - return -1; - } - } - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_42) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_46) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - if (condval_12 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_13; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_13); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_47) != 0) { - return -1; - } - } - int32_t condval_14; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[0]); - } - if ((condval_14 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_48) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_49) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_51) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_52) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_53) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_54) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - if (condval_15 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_16; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_16); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_56) != 0) { - return -1; - } - } - int32_t condval_17; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[0]); - } - if ((condval_17 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_58) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_60) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_61) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_62) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_63) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1]) != 16) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_64) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - if (condval_18 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[0]); - } - if ((condval_20 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_66) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_67) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_68) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_69) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_70) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)layer_input)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)layer_input)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)layer_input)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_71) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_72) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_73) != 0) { - return -1; - } - } - int32_t condval_21; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - if (condval_21 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_22; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_22); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - int32_t condval_23; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[0]); - } - if ((condval_23 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_75) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)layer_input)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_76) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_77) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_78) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_79) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)norm_weight)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)norm_weight)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)norm_weight)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_80) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - if (condval_24 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_25; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_25); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)norm_weight)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_83) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_85) != 0) { - return -1; - } - } - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_86) != 0) { - return -1; - } - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_87) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = gemm_out_mul_1; - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = gemm_out_sqrsum_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = hc_scale_1; - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = layer_input_1; - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = norm_weight_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = post_mix_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)96); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)29040); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = (int64_t)0; - if (mhc_pre_big_fuse_with_norm_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_pre_big_fuse_with_norm_tilelang_kernel", &mhc_pre_big_fuse_with_norm_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_pre_big_fuse_with_norm_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 15, &result_88) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/455ef1b63b3a592f4741c938fcfb8d01f90e8269880307cffbf4438ed668de0b/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/455ef1b63b3a592f4741c938fcfb8d01f90e8269880307cffbf4438ed668de0b/params.pkl deleted file mode 100644 index 83d199651dbc9eb2aeb1a826784965864a7b0ee1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2200 zcmeHIO=}x55KZfkE~E*DQknz`txhhOrAeR%pL-JxAt}8SBedSJrFK_Zv~rqa7MeqP zQchNWU;j#H*LIpFh8SO+y=ize@6DSX=~v_WD%hyLo^QLCRwj^eH4H%wrGhZyhNs?b zQZNdV_hRB9PlN?UHFMuvl;=oLPX=njk`QJW4^G(~W4 zW@1aU+Tz9k{)$&qXZOVGscGWvI=Hp6SG2}gwVCeRO%0-It@c06*J?hZI&J-1foeoz u%?3p4pEg=6PpQ3rc0GDkUaw2tl=uFt$2~0HsD;nUf-PL$ElkEM55m8rk@gV) diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/47c189497f8ff487a170b06db17717920a83856d30485b87094ce9d42596885b/device_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/47c189497f8ff487a170b06db17717920a83856d30485b87094ce9d42596885b/device_kernel.cu deleted file mode 100644 index 1e15dbf..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/47c189497f8ff487a170b06db17717920a83856d30485b87094ce9d42596885b/device_kernel.cu +++ /dev/null @@ -1,424 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float mixes[1]; - float rms[1]; - float cm[1]; - float row_max[1]; - float row_sum[1]; - float col_sum[1]; - float sumsq_per_pos[16]; - float sumsq[1]; - float rsqrt_norm[1]; - float xl[64]; - float ol[16]; - bfloat16_t xs_local_cast[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t xs_local_cast_2[8]; - float w_local[16]; - float ol_1[16]; - bfloat16_t w_shared_local_cast_3[8]; - bfloat16_t output_shared_local_cast_4[8]; - bfloat16_t layer_input_local_cast_5[8]; - bfloat16_t w_shared_local_cast_6[8]; - bfloat16_t output_shared_local_cast_7[8]; - bfloat16_t layer_input_local_cast_8[8]; - bfloat16_t w_shared_local_cast_9[8]; - bfloat16_t output_shared_local_cast_10[8]; - bfloat16_t layer_input_local_cast_11[8]; - mixes[0] = 0x0p+0f/*0.000000e+00*/; - rms[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - for (int i_split = 0; i_split < 4; ++i_split) { - rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]); - } - rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - mixes[0] = 0x0p+0f/*0.000000e+00*/; - for (int i_split_1 = 0; i_split_1 < 4; ++i_split_1) { - mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]); - } - mixes[0] = (mixes[0] * rms[0]); - if ((((int)threadIdx.x) / 24) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0]; - } - __syncthreads(); - if (((int)threadIdx.x) < 32) { - if (((int)threadIdx.x) < 4) { - post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/); - } - cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]); - row_max[0] = -CUDART_INF_F; - row_max[0] = max(row_max[0], cm[0]); - row_max[0] = tl::AllReduce>::run(row_max[0]); - cm[0] = expf((cm[0] - row_max[0])); - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - for (int __1 = 0; __1 < 19; ++__1) { - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - } - if ((((int)threadIdx.x) >> 4) == 0) { - comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0]; - } - } else { - if (((int)threadIdx.x) < 36) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - #pragma unroll - for (int i = 0; i < 4; ++i) { - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_2 = 0; i_2 < 8; ++i_2) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_4 = 0; i_4 < 8; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - tl::__sync_thread_partial<3, 64>(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)]; - #pragma unroll - for (int i_6 = 0; i_6 < 16; ++i_6) { - ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)])); - } - } - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - float4 __3; - float4 v__1 = *(float4*)(sumsq_per_pos + (i_7 * 4)); - float4 __4; - float4 v__2 = *(float4*)(ol + (i_7 * 4)); - __4.x = (v__2.x*v__2.x); - __4.y = (v__2.y*v__2.y); - __4.z = (v__2.z*v__2.z); - __4.w = (v__2.w*v__2.w); - __3.x = (v__1.x+__4.x); - __3.y = (v__1.y+__4.y); - __3.z = (v__1.z+__4.z); - __3.w = (v__1.w+__4.w); - *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3; - uint2 __5; - float4 v__3 = *(float4*)(ol + (i_7 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __5; - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_8 = 0; i_8 < 8; ++i_8) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __6; - uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]); - *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __6; - } - } - #pragma unroll - for (int i_9 = 0; i_9 < 4; ++i_9) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)]; - #pragma unroll - for (int i_10 = 0; i_10 < 16; ++i_10) { - ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)])); - } - } - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - float4 __7; - float4 v__5 = *(float4*)(sumsq_per_pos + (i_11 * 4)); - float4 __8; - float4 v__6 = *(float4*)(ol + (i_11 * 4)); - __8.x = (v__6.x*v__6.x); - __8.y = (v__6.y*v__6.y); - __8.z = (v__6.z*v__6.z); - __8.w = (v__6.w*v__6.w); - __7.x = (v__5.x+__8.x); - __7.y = (v__5.y+__8.y); - __7.z = (v__5.z+__8.z); - __7.w = (v__5.w+__8.w); - *(float4*)(sumsq_per_pos + (i_11 * 4)) = __7; - uint2 __9; - float4 v__7 = *(float4*)(ol + (i_11 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__9))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__9))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __9; - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_12 = 0; i_12 < 8; ++i_12) { - *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __10; - uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4)); - ((float2*)(&__10))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]); - ((float2*)(&__10))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]); - *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __10; - } - } - #pragma unroll - for (int i_13 = 0; i_13 < 4; ++i_13) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)]; - #pragma unroll - for (int i_14 = 0; i_14 < 16; ++i_14) { - ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)])); - } - } - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - float4 __11; - float4 v__9 = *(float4*)(sumsq_per_pos + (i_15 * 4)); - float4 __12; - float4 v__10 = *(float4*)(ol + (i_15 * 4)); - __12.x = (v__10.x*v__10.x); - __12.y = (v__10.y*v__10.y); - __12.z = (v__10.z*v__10.z); - __12.w = (v__10.w*v__10.w); - __11.x = (v__9.x+__12.x); - __11.y = (v__9.y+__12.y); - __11.z = (v__9.z+__12.z); - __11.w = (v__9.w+__12.w); - *(float4*)(sumsq_per_pos + (i_15 * 4)) = __11; - uint2 __13; - float4 v__11 = *(float4*)(ol + (i_15 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__13))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__13))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __13; - } - sumsq[0] = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int rv = 0; rv < 16; ++rv) { - sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]); - } - tl::__sync_thread_partial<3, 64>(); - sumsq[0] = tl::AllReduce>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192]))); - rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_17 = 0; i_17 < 2; ++i_17) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)]))); - } - tl::cp_async_commit(); - for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_18 = 0; i_18 < 2; ++i_18) { - *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_3 = 0; vec_3 < 2; ++vec_3) { - float4 __14; - uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4)); - ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]); - ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]); - *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __14; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_19 = 0; i_19 < 2; ++i_19) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)]))); - } - tl::cp_async_commit(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_20 = 0; i_20 < 2; ++i_20) { - *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984)); - for (int vec_4 = 0; vec_4 < 2; ++vec_4) { - float4 __15; - float4 __16; - float4 __17; - uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4)); - ((float2*)(&__17))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]); - ((float2*)(&__17))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]); - float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __16.x = (__17.x*v__14.x); - __16.y = (__17.y*v__14.y); - __16.z = (__17.z*v__14.z); - __16.w = (__17.w*v__14.w); - float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4))); - __15.x = (__16.x*v__15.x); - __15.y = (__16.y*v__15.y); - __15.z = (__16.z*v__15.z); - __15.w = (__16.w*v__15.w); - *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __15; - } - } - #pragma unroll - for (int i_21 = 0; i_21 < 2; ++i_21) { - for (int vec_5 = 0; vec_5 < 2; ++vec_5) { - uint2 __18; - float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__18))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__18))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]); - *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __18; - } - *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0); - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_22 = 0; i_22 < 2; ++i_22) { - *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_6 = 0; vec_6 < 2; ++vec_6) { - float4 __19; - uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4)); - ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]); - ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]); - *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __19; - } - } - #pragma unroll - for (int i_23 = 0; i_23 < 2; ++i_23) { - *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032)); - for (int vec_7 = 0; vec_7 < 2; ++vec_7) { - float4 __20; - float4 __21; - float4 __22; - uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4)); - ((float2*)(&__22))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]); - ((float2*)(&__22))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]); - float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __21.x = (__22.x*v__19.x); - __21.y = (__22.y*v__19.y); - __21.z = (__22.z*v__19.z); - __21.w = (__22.w*v__19.w); - float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4))); - __20.x = (__21.x*v__20.x); - __20.y = (__21.y*v__20.y); - __20.z = (__21.z*v__20.z); - __20.w = (__21.w*v__20.w); - *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __20; - } - } - #pragma unroll - for (int i_24 = 0; i_24 < 2; ++i_24) { - for (int vec_8 = 0; vec_8 < 2; ++vec_8) { - uint2 __23; - float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__23))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__23))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]); - *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __23; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0); - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_25 = 0; i_25 < 2; ++i_25) { - *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104)); - for (int vec_9 = 0; vec_9 < 2; ++vec_9) { - float4 __24; - uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4)); - ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]); - ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]); - *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __24; - } - } - #pragma unroll - for (int i_26 = 0; i_26 < 2; ++i_26) { - *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056)); - for (int vec_10 = 0; vec_10 < 2; ++vec_10) { - float4 __25; - float4 __26; - float4 __27; - uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4)); - ((float2*)(&__27))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]); - ((float2*)(&__27))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]); - float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __26.x = (__27.x*v__24.x); - __26.y = (__27.y*v__24.y); - __26.z = (__27.z*v__24.z); - __26.w = (__27.w*v__24.w); - float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4))); - __25.x = (__26.x*v__25.x); - __25.y = (__26.y*v__25.y); - __25.z = (__26.z*v__25.z); - __25.w = (__26.w*v__25.w); - *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __25; - } - } - #pragma unroll - for (int i_27 = 0; i_27 < 2; ++i_27) { - for (int vec_11 = 0; vec_11 < 2; ++vec_11) { - uint2 __28; - float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__28))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__28))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]); - *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __28; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/47c189497f8ff487a170b06db17717920a83856d30485b87094ce9d42596885b/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/47c189497f8ff487a170b06db17717920a83856d30485b87094ce9d42596885b/host_kernel.cu deleted file mode 100644 index 825ac07..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/47c189497f8ff487a170b06db17717920a83856d30485b87094ce9d42596885b/host_kernel.cu +++ /dev/null @@ -1,3359 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_pre_big_fuse_with_norm_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[16]; - void* stack_ffi_any = stack; - if (!((num_args == 9))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_pre_big_fuse_with_norm_tilelang: num_args should be 9", (long long)(num_args), (long long)(9)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang: args pointer is NULL"); - return -1; - } - int32_t gemm_out_mul_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((gemm_out_mul_type_index == 0) || (gemm_out_mul_type_index == 4)) || (gemm_out_mul_type_index == 7)) || (64 <= gemm_out_mul_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_mul expected pointer or tensor handle"); - return -1; - } - int32_t gemm_out_sqrsum_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((gemm_out_sqrsum_type_index == 0) || (gemm_out_sqrsum_type_index == 4)) || (gemm_out_sqrsum_type_index == 7)) || (64 <= gemm_out_sqrsum_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_sqrsum expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t layer_input_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((layer_input_type_index == 0) || (layer_input_type_index == 4)) || (layer_input_type_index == 7)) || (64 <= layer_input_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input layer_input expected pointer or tensor handle"); - return -1; - } - int32_t norm_weight_type_index = (((TVMFFIAny*)args)[8].type_index); - if (!(((((norm_weight_type_index == 0) || (norm_weight_type_index == 4)) || (norm_weight_type_index == 7)) || (64 <= norm_weight_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input norm_weight expected pointer or tensor handle"); - return -1; - } - void* gemm_out_mul = ((gemm_out_mul_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* gemm_out_sqrsum = ((gemm_out_sqrsum_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* layer_input = ((layer_input_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - void* norm_weight = ((norm_weight_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[8].v_ptr) + 24)) : (((TVMFFIAny*)args)[8].v_ptr)); - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null = (gemm_out_mul == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null = (gemm_out_sqrsum == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_residual_is_null = (residual == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null = (layer_input == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.layer_input is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null = (norm_weight == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.norm_weight is expected to have non-NULL pointer"); - return -1; - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape = (((DLTensor*)gemm_out_mul)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape = (((DLTensor*)gemm_out_sqrsum)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape = (((DLTensor*)layer_input)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape = (((DLTensor*)norm_weight)[0].shape); - if ((((DLTensor*)gemm_out_mul)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul, mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum, mhc_pre_big_fuse_with_norm_tilelang.residual, mhc_pre_big_fuse_with_norm_tilelang.post_mix, mhc_pre_big_fuse_with_norm_tilelang.comb_mix, mhc_pre_big_fuse_with_norm_tilelang.layer_input"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides = (((DLTensor*)gemm_out_mul)[0].strides); - int32_t dev_id = (((DLTensor*)gemm_out_mul)[0].device.device_id); - void* gemm_out_mul_1 = (((DLTensor*)gemm_out_mul)[0].data); - if ((((DLTensor*)gemm_out_sqrsum)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides = (((DLTensor*)gemm_out_sqrsum)[0].strides); - void* gemm_out_sqrsum_1 = (((DLTensor*)gemm_out_sqrsum)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)comb_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)layer_input)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides = (((DLTensor*)layer_input)[0].strides); - void* layer_input_1 = (((DLTensor*)layer_input)[0].data); - if ((((DLTensor*)norm_weight)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_9) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides = (((DLTensor*)norm_weight)[0].strides); - void* norm_weight_1 = (((DLTensor*)norm_weight)[0].data); - if ((((((DLTensor*)gemm_out_mul)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_mul)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_mul)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[1]); - } - if ((condval_2 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - int32_t condval_3; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[0]); - } - if (((num_tokens * 24) != condval_3) && (num_tokens != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_15) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_mul)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_mul)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_17) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_18) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)gemm_out_sqrsum)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[1]); - } - if ((condval_4 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_22) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[0]); - } - if ((((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != condval_5) && (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_23) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_sqrsum)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_26) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_27) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_28) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_29) != 0) { - return -1; - } - } - int32_t condval_6; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - if (condval_6 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_7; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_7); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_34) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_35) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_36) != 0) { - return -1; - } - } - int32_t condval_8; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - if (condval_8 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_9; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_9); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_38) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_40) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_41) != 0) { - return -1; - } - } - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_42) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_46) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - if (condval_12 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_13; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_13); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_47) != 0) { - return -1; - } - } - int32_t condval_14; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[0]); - } - if ((condval_14 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_48) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_49) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_51) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_52) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_53) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_54) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - if (condval_15 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_16; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_16); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_56) != 0) { - return -1; - } - } - int32_t condval_17; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[0]); - } - if ((condval_17 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_58) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_60) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_61) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_62) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_63) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1]) != 16) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_64) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - if (condval_18 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[0]); - } - if ((condval_20 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_66) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_67) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_68) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_69) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_70) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)layer_input)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)layer_input)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)layer_input)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_71) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_72) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_73) != 0) { - return -1; - } - } - int32_t condval_21; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - if (condval_21 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_22; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_22); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - int32_t condval_23; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[0]); - } - if ((condval_23 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_75) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)layer_input)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_76) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_77) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_78) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_79) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)norm_weight)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)norm_weight)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)norm_weight)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_80) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - if (condval_24 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_25; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_25); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)norm_weight)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_83) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_85) != 0) { - return -1; - } - } - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_86) != 0) { - return -1; - } - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_87) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = gemm_out_mul_1; - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = gemm_out_sqrsum_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = hc_scale_1; - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = layer_input_1; - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = norm_weight_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = post_mix_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)96); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)29040); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = (int64_t)0; - if (mhc_pre_big_fuse_with_norm_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_pre_big_fuse_with_norm_tilelang_kernel", &mhc_pre_big_fuse_with_norm_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_pre_big_fuse_with_norm_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 15, &result_88) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/47c189497f8ff487a170b06db17717920a83856d30485b87094ce9d42596885b/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/47c189497f8ff487a170b06db17717920a83856d30485b87094ce9d42596885b/params.pkl deleted file mode 100644 index 14a52c912ac3ed5c0c1e83b8998f20195bf69b51..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2200 zcmeHIO=}x55KZHc8r%j$2uTZpRwoC{lEjdM&$)?)(3W085L)lpQnD*8S~*QI3(X-t zsVA$yuYaYp>yM^MputyXZyMgrd-G;T`d<6>v$0lw=f3T>tc)S%Y8ZeTN(Dj64Ntsl zQZNeQ)ABU&dpABubS<*qQt)oWUL|ZWkO7+pM1A7j%R#KU?H>ADf8$SbM6nrchH|gw zX?<<5${W@5vHJAx4d`2Ym@DyNg8fQzMf?{sKBCS;`gZ z?PKnYKq~Z)fmU;R`U^6Y3uI_lp_)= uHXvI4u-007O6}FN>)ET~dVSJO`QX2L-oxUJdh}U&WDB3}7AE7B2f-g}G4=@n diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/596345ac7ba68225bdd1b9b6abe5e4fbb63f7c57ec2523668ca14b62460b457a/device_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/596345ac7ba68225bdd1b9b6abe5e4fbb63f7c57ec2523668ca14b62460b457a/device_kernel.cu deleted file mode 100644 index ee7574e..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/596345ac7ba68225bdd1b9b6abe5e4fbb63f7c57ec2523668ca14b62460b457a/device_kernel.cu +++ /dev/null @@ -1,424 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float mixes[1]; - float rms[1]; - float cm[1]; - float row_max[1]; - float row_sum[1]; - float col_sum[1]; - float sumsq_per_pos[16]; - float sumsq[1]; - float rsqrt_norm[1]; - float xl[64]; - float ol[16]; - bfloat16_t xs_local_cast[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t xs_local_cast_2[8]; - float w_local[16]; - float ol_1[16]; - bfloat16_t w_shared_local_cast_3[8]; - bfloat16_t output_shared_local_cast_4[8]; - bfloat16_t layer_input_local_cast_5[8]; - bfloat16_t w_shared_local_cast_6[8]; - bfloat16_t output_shared_local_cast_7[8]; - bfloat16_t layer_input_local_cast_8[8]; - bfloat16_t w_shared_local_cast_9[8]; - bfloat16_t output_shared_local_cast_10[8]; - bfloat16_t layer_input_local_cast_11[8]; - mixes[0] = 0x0p+0f/*0.000000e+00*/; - rms[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - for (int i_split = 0; i_split < 2; ++i_split) { - rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]); - } - rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - mixes[0] = 0x0p+0f/*0.000000e+00*/; - for (int i_split_1 = 0; i_split_1 < 2; ++i_split_1) { - mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]); - } - mixes[0] = (mixes[0] * rms[0]); - if ((((int)threadIdx.x) / 24) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0]; - } - __syncthreads(); - if (((int)threadIdx.x) < 32) { - if (((int)threadIdx.x) < 4) { - post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/); - } - cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]); - row_max[0] = -CUDART_INF_F; - row_max[0] = max(row_max[0], cm[0]); - row_max[0] = tl::AllReduce>::run(row_max[0]); - cm[0] = expf((cm[0] - row_max[0])); - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - for (int __1 = 0; __1 < 19; ++__1) { - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - } - if ((((int)threadIdx.x) >> 4) == 0) { - comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0]; - } - } else { - if (((int)threadIdx.x) < 36) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - #pragma unroll - for (int i = 0; i < 4; ++i) { - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_2 = 0; i_2 < 8; ++i_2) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_4 = 0; i_4 < 8; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - tl::__sync_thread_partial<3, 64>(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)]; - #pragma unroll - for (int i_6 = 0; i_6 < 16; ++i_6) { - ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)])); - } - } - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - float4 __3; - float4 v__1 = *(float4*)(sumsq_per_pos + (i_7 * 4)); - float4 __4; - float4 v__2 = *(float4*)(ol + (i_7 * 4)); - __4.x = (v__2.x*v__2.x); - __4.y = (v__2.y*v__2.y); - __4.z = (v__2.z*v__2.z); - __4.w = (v__2.w*v__2.w); - __3.x = (v__1.x+__4.x); - __3.y = (v__1.y+__4.y); - __3.z = (v__1.z+__4.z); - __3.w = (v__1.w+__4.w); - *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3; - uint2 __5; - float4 v__3 = *(float4*)(ol + (i_7 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __5; - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_8 = 0; i_8 < 8; ++i_8) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __6; - uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]); - *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __6; - } - } - #pragma unroll - for (int i_9 = 0; i_9 < 4; ++i_9) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)]; - #pragma unroll - for (int i_10 = 0; i_10 < 16; ++i_10) { - ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)])); - } - } - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - float4 __7; - float4 v__5 = *(float4*)(sumsq_per_pos + (i_11 * 4)); - float4 __8; - float4 v__6 = *(float4*)(ol + (i_11 * 4)); - __8.x = (v__6.x*v__6.x); - __8.y = (v__6.y*v__6.y); - __8.z = (v__6.z*v__6.z); - __8.w = (v__6.w*v__6.w); - __7.x = (v__5.x+__8.x); - __7.y = (v__5.y+__8.y); - __7.z = (v__5.z+__8.z); - __7.w = (v__5.w+__8.w); - *(float4*)(sumsq_per_pos + (i_11 * 4)) = __7; - uint2 __9; - float4 v__7 = *(float4*)(ol + (i_11 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__9))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__9))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __9; - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_12 = 0; i_12 < 8; ++i_12) { - *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __10; - uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4)); - ((float2*)(&__10))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]); - ((float2*)(&__10))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]); - *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __10; - } - } - #pragma unroll - for (int i_13 = 0; i_13 < 4; ++i_13) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)]; - #pragma unroll - for (int i_14 = 0; i_14 < 16; ++i_14) { - ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)])); - } - } - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - float4 __11; - float4 v__9 = *(float4*)(sumsq_per_pos + (i_15 * 4)); - float4 __12; - float4 v__10 = *(float4*)(ol + (i_15 * 4)); - __12.x = (v__10.x*v__10.x); - __12.y = (v__10.y*v__10.y); - __12.z = (v__10.z*v__10.z); - __12.w = (v__10.w*v__10.w); - __11.x = (v__9.x+__12.x); - __11.y = (v__9.y+__12.y); - __11.z = (v__9.z+__12.z); - __11.w = (v__9.w+__12.w); - *(float4*)(sumsq_per_pos + (i_15 * 4)) = __11; - uint2 __13; - float4 v__11 = *(float4*)(ol + (i_15 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__13))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__13))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __13; - } - sumsq[0] = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int rv = 0; rv < 16; ++rv) { - sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]); - } - tl::__sync_thread_partial<3, 64>(); - sumsq[0] = tl::AllReduce>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192]))); - rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_17 = 0; i_17 < 2; ++i_17) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)]))); - } - tl::cp_async_commit(); - for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_18 = 0; i_18 < 2; ++i_18) { - *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_3 = 0; vec_3 < 2; ++vec_3) { - float4 __14; - uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4)); - ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]); - ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]); - *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __14; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_19 = 0; i_19 < 2; ++i_19) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)]))); - } - tl::cp_async_commit(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_20 = 0; i_20 < 2; ++i_20) { - *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984)); - for (int vec_4 = 0; vec_4 < 2; ++vec_4) { - float4 __15; - float4 __16; - float4 __17; - uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4)); - ((float2*)(&__17))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]); - ((float2*)(&__17))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]); - float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __16.x = (__17.x*v__14.x); - __16.y = (__17.y*v__14.y); - __16.z = (__17.z*v__14.z); - __16.w = (__17.w*v__14.w); - float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4))); - __15.x = (__16.x*v__15.x); - __15.y = (__16.y*v__15.y); - __15.z = (__16.z*v__15.z); - __15.w = (__16.w*v__15.w); - *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __15; - } - } - #pragma unroll - for (int i_21 = 0; i_21 < 2; ++i_21) { - for (int vec_5 = 0; vec_5 < 2; ++vec_5) { - uint2 __18; - float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__18))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__18))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]); - *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __18; - } - *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0); - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_22 = 0; i_22 < 2; ++i_22) { - *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_6 = 0; vec_6 < 2; ++vec_6) { - float4 __19; - uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4)); - ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]); - ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]); - *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __19; - } - } - #pragma unroll - for (int i_23 = 0; i_23 < 2; ++i_23) { - *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032)); - for (int vec_7 = 0; vec_7 < 2; ++vec_7) { - float4 __20; - float4 __21; - float4 __22; - uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4)); - ((float2*)(&__22))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]); - ((float2*)(&__22))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]); - float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __21.x = (__22.x*v__19.x); - __21.y = (__22.y*v__19.y); - __21.z = (__22.z*v__19.z); - __21.w = (__22.w*v__19.w); - float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4))); - __20.x = (__21.x*v__20.x); - __20.y = (__21.y*v__20.y); - __20.z = (__21.z*v__20.z); - __20.w = (__21.w*v__20.w); - *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __20; - } - } - #pragma unroll - for (int i_24 = 0; i_24 < 2; ++i_24) { - for (int vec_8 = 0; vec_8 < 2; ++vec_8) { - uint2 __23; - float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__23))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__23))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]); - *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __23; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0); - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_25 = 0; i_25 < 2; ++i_25) { - *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104)); - for (int vec_9 = 0; vec_9 < 2; ++vec_9) { - float4 __24; - uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4)); - ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]); - ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]); - *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __24; - } - } - #pragma unroll - for (int i_26 = 0; i_26 < 2; ++i_26) { - *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056)); - for (int vec_10 = 0; vec_10 < 2; ++vec_10) { - float4 __25; - float4 __26; - float4 __27; - uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4)); - ((float2*)(&__27))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]); - ((float2*)(&__27))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]); - float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __26.x = (__27.x*v__24.x); - __26.y = (__27.y*v__24.y); - __26.z = (__27.z*v__24.z); - __26.w = (__27.w*v__24.w); - float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4))); - __25.x = (__26.x*v__25.x); - __25.y = (__26.y*v__25.y); - __25.z = (__26.z*v__25.z); - __25.w = (__26.w*v__25.w); - *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __25; - } - } - #pragma unroll - for (int i_27 = 0; i_27 < 2; ++i_27) { - for (int vec_11 = 0; vec_11 < 2; ++vec_11) { - uint2 __28; - float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__28))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__28))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]); - *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __28; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/596345ac7ba68225bdd1b9b6abe5e4fbb63f7c57ec2523668ca14b62460b457a/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/596345ac7ba68225bdd1b9b6abe5e4fbb63f7c57ec2523668ca14b62460b457a/host_kernel.cu deleted file mode 100644 index 59d8e6b..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/596345ac7ba68225bdd1b9b6abe5e4fbb63f7c57ec2523668ca14b62460b457a/host_kernel.cu +++ /dev/null @@ -1,3359 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_pre_big_fuse_with_norm_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[16]; - void* stack_ffi_any = stack; - if (!((num_args == 9))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_pre_big_fuse_with_norm_tilelang: num_args should be 9", (long long)(num_args), (long long)(9)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang: args pointer is NULL"); - return -1; - } - int32_t gemm_out_mul_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((gemm_out_mul_type_index == 0) || (gemm_out_mul_type_index == 4)) || (gemm_out_mul_type_index == 7)) || (64 <= gemm_out_mul_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_mul expected pointer or tensor handle"); - return -1; - } - int32_t gemm_out_sqrsum_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((gemm_out_sqrsum_type_index == 0) || (gemm_out_sqrsum_type_index == 4)) || (gemm_out_sqrsum_type_index == 7)) || (64 <= gemm_out_sqrsum_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_sqrsum expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t layer_input_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((layer_input_type_index == 0) || (layer_input_type_index == 4)) || (layer_input_type_index == 7)) || (64 <= layer_input_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input layer_input expected pointer or tensor handle"); - return -1; - } - int32_t norm_weight_type_index = (((TVMFFIAny*)args)[8].type_index); - if (!(((((norm_weight_type_index == 0) || (norm_weight_type_index == 4)) || (norm_weight_type_index == 7)) || (64 <= norm_weight_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input norm_weight expected pointer or tensor handle"); - return -1; - } - void* gemm_out_mul = ((gemm_out_mul_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* gemm_out_sqrsum = ((gemm_out_sqrsum_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* layer_input = ((layer_input_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - void* norm_weight = ((norm_weight_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[8].v_ptr) + 24)) : (((TVMFFIAny*)args)[8].v_ptr)); - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null = (gemm_out_mul == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null = (gemm_out_sqrsum == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_residual_is_null = (residual == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null = (layer_input == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.layer_input is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null = (norm_weight == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.norm_weight is expected to have non-NULL pointer"); - return -1; - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape = (((DLTensor*)gemm_out_mul)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape = (((DLTensor*)gemm_out_sqrsum)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape = (((DLTensor*)layer_input)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape = (((DLTensor*)norm_weight)[0].shape); - if ((((DLTensor*)gemm_out_mul)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul, mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum, mhc_pre_big_fuse_with_norm_tilelang.residual, mhc_pre_big_fuse_with_norm_tilelang.post_mix, mhc_pre_big_fuse_with_norm_tilelang.comb_mix, mhc_pre_big_fuse_with_norm_tilelang.layer_input"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides = (((DLTensor*)gemm_out_mul)[0].strides); - int32_t dev_id = (((DLTensor*)gemm_out_mul)[0].device.device_id); - void* gemm_out_mul_1 = (((DLTensor*)gemm_out_mul)[0].data); - if ((((DLTensor*)gemm_out_sqrsum)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides = (((DLTensor*)gemm_out_sqrsum)[0].strides); - void* gemm_out_sqrsum_1 = (((DLTensor*)gemm_out_sqrsum)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)comb_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)layer_input)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides = (((DLTensor*)layer_input)[0].strides); - void* layer_input_1 = (((DLTensor*)layer_input)[0].data); - if ((((DLTensor*)norm_weight)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_9) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides = (((DLTensor*)norm_weight)[0].strides); - void* norm_weight_1 = (((DLTensor*)norm_weight)[0].data); - if ((((((DLTensor*)gemm_out_mul)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_mul)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_mul)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0]) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[1]); - } - if ((condval_2 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - int32_t condval_3; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[0]); - } - if (((num_tokens * 24) != condval_3) && (num_tokens != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_15) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_mul)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_mul)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_17) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_18) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)gemm_out_sqrsum)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0]) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[1]); - } - if ((condval_4 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_22) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[0]); - } - if ((((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != condval_5) && (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_23) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_sqrsum)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_26) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_27) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_28) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_29) != 0) { - return -1; - } - } - int32_t condval_6; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - if (condval_6 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_7; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_7); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_34) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_35) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_36) != 0) { - return -1; - } - } - int32_t condval_8; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - if (condval_8 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_9; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_9); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_38) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_40) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_41) != 0) { - return -1; - } - } - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_42) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_46) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - if (condval_12 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_13; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_13); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_47) != 0) { - return -1; - } - } - int32_t condval_14; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[0]); - } - if ((condval_14 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_48) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_49) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_51) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_52) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_53) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_54) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - if (condval_15 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_16; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_16); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_56) != 0) { - return -1; - } - } - int32_t condval_17; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[0]); - } - if ((condval_17 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_58) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_60) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_61) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_62) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_63) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1]) != 16) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_64) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - if (condval_18 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[0]); - } - if ((condval_20 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_66) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_67) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_68) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_69) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_70) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)layer_input)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)layer_input)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)layer_input)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_71) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_72) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_73) != 0) { - return -1; - } - } - int32_t condval_21; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - if (condval_21 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_22; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_22); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - int32_t condval_23; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[0]); - } - if ((condval_23 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_75) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)layer_input)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_76) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_77) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_78) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_79) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)norm_weight)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)norm_weight)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)norm_weight)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_80) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - if (condval_24 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_25; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_25); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)norm_weight)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_83) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_85) != 0) { - return -1; - } - } - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_86) != 0) { - return -1; - } - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_87) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = gemm_out_mul_1; - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = gemm_out_sqrsum_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = hc_scale_1; - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = layer_input_1; - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = norm_weight_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = post_mix_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)96); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)29040); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = (int64_t)0; - if (mhc_pre_big_fuse_with_norm_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_pre_big_fuse_with_norm_tilelang_kernel", &mhc_pre_big_fuse_with_norm_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_pre_big_fuse_with_norm_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 15, &result_88) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/596345ac7ba68225bdd1b9b6abe5e4fbb63f7c57ec2523668ca14b62460b457a/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/596345ac7ba68225bdd1b9b6abe5e4fbb63f7c57ec2523668ca14b62460b457a/params.pkl deleted file mode 100644 index 6dd6dc4039d73297070ae043b3abb880edfae64c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2200 zcmeHIO-my|5Os}-0Y38wf^SP$;0 z_muYc`B%1PlDMu4E4jwGb@8g+t5;LqKdbk@>vQFI>6>od${1pCs13ba?8chgqeFk^Z~aM*C^m+TQEqvX z*3ky5Y_X03Zh>{mDy(aC!fZc{!4Hw_8M=PVEk7^lhn~*UkMx7}XFmJoLs!iN$2j&w z=f$d9VhXO94!=RiBDgFRSZM;dPK|e!3vMuchuJ8yCb)`_(LD@^(b_VpBDf~)qZUy* zf{e7k4oJ~8X_FJJfaI;mbK$@jYozLt7KukP7`H`4fslHtPL<63XQXiacnQY8h8bTW+OBhQmC+%zYNpP3VWBG8(3+iTaC~ zi51akinV|J6&oXGx5duLG_gAiZcXhKt?+4Wrkl@G>tVT82cITuH5t(YZTww -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float mixes[1]; - float rms[1]; - float cm[1]; - float row_max[1]; - float row_sum[1]; - float col_sum[1]; - float sumsq_per_pos[16]; - float sumsq[1]; - float rsqrt_norm[1]; - float xl[64]; - float ol[16]; - bfloat16_t xs_local_cast[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t xs_local_cast_2[8]; - float w_local[16]; - float ol_1[16]; - bfloat16_t w_shared_local_cast_3[8]; - bfloat16_t output_shared_local_cast_4[8]; - bfloat16_t layer_input_local_cast_5[8]; - bfloat16_t w_shared_local_cast_6[8]; - bfloat16_t output_shared_local_cast_7[8]; - bfloat16_t layer_input_local_cast_8[8]; - bfloat16_t w_shared_local_cast_9[8]; - bfloat16_t output_shared_local_cast_10[8]; - bfloat16_t layer_input_local_cast_11[8]; - mixes[0] = 0x0p+0f/*0.000000e+00*/; - rms[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - rms[0] = (rms[0] + gemm_out_sqrsum[((int64_t)((int)blockIdx.x))]); - rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - mixes[0] = 0x0p+0f/*0.000000e+00*/; - mixes[0] = (mixes[0] + gemm_out_mul[((((int64_t)((int)blockIdx.x)) * (int64_t)24) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]); - mixes[0] = (mixes[0] * rms[0]); - if ((((int)threadIdx.x) / 24) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0]; - } - __syncthreads(); - if (((int)threadIdx.x) < 32) { - if (((int)threadIdx.x) < 4) { - post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/); - } - cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]); - row_max[0] = -CUDART_INF_F; - row_max[0] = max(row_max[0], cm[0]); - row_max[0] = tl::AllReduce>::run(row_max[0]); - cm[0] = expf((cm[0] - row_max[0])); - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - for (int __1 = 0; __1 < 19; ++__1) { - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - } - if ((((int)threadIdx.x) >> 4) == 0) { - comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0]; - } - } else { - if (((int)threadIdx.x) < 36) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - #pragma unroll - for (int i = 0; i < 4; ++i) { - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_2 = 0; i_2 < 8; ++i_2) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_4 = 0; i_4 < 8; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - tl::__sync_thread_partial<3, 64>(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)]; - #pragma unroll - for (int i_6 = 0; i_6 < 16; ++i_6) { - ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)])); - } - } - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - float4 __3; - float4 v__1 = *(float4*)(sumsq_per_pos + (i_7 * 4)); - float4 __4; - float4 v__2 = *(float4*)(ol + (i_7 * 4)); - __4.x = (v__2.x*v__2.x); - __4.y = (v__2.y*v__2.y); - __4.z = (v__2.z*v__2.z); - __4.w = (v__2.w*v__2.w); - __3.x = (v__1.x+__4.x); - __3.y = (v__1.y+__4.y); - __3.z = (v__1.z+__4.z); - __3.w = (v__1.w+__4.w); - *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3; - uint2 __5; - float4 v__3 = *(float4*)(ol + (i_7 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __5; - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_8 = 0; i_8 < 8; ++i_8) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __6; - uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]); - *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __6; - } - } - #pragma unroll - for (int i_9 = 0; i_9 < 4; ++i_9) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)]; - #pragma unroll - for (int i_10 = 0; i_10 < 16; ++i_10) { - ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)])); - } - } - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - float4 __7; - float4 v__5 = *(float4*)(sumsq_per_pos + (i_11 * 4)); - float4 __8; - float4 v__6 = *(float4*)(ol + (i_11 * 4)); - __8.x = (v__6.x*v__6.x); - __8.y = (v__6.y*v__6.y); - __8.z = (v__6.z*v__6.z); - __8.w = (v__6.w*v__6.w); - __7.x = (v__5.x+__8.x); - __7.y = (v__5.y+__8.y); - __7.z = (v__5.z+__8.z); - __7.w = (v__5.w+__8.w); - *(float4*)(sumsq_per_pos + (i_11 * 4)) = __7; - uint2 __9; - float4 v__7 = *(float4*)(ol + (i_11 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__9))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__9))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __9; - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_12 = 0; i_12 < 8; ++i_12) { - *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __10; - uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4)); - ((float2*)(&__10))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]); - ((float2*)(&__10))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]); - *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __10; - } - } - #pragma unroll - for (int i_13 = 0; i_13 < 4; ++i_13) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)]; - #pragma unroll - for (int i_14 = 0; i_14 < 16; ++i_14) { - ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)])); - } - } - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - float4 __11; - float4 v__9 = *(float4*)(sumsq_per_pos + (i_15 * 4)); - float4 __12; - float4 v__10 = *(float4*)(ol + (i_15 * 4)); - __12.x = (v__10.x*v__10.x); - __12.y = (v__10.y*v__10.y); - __12.z = (v__10.z*v__10.z); - __12.w = (v__10.w*v__10.w); - __11.x = (v__9.x+__12.x); - __11.y = (v__9.y+__12.y); - __11.z = (v__9.z+__12.z); - __11.w = (v__9.w+__12.w); - *(float4*)(sumsq_per_pos + (i_15 * 4)) = __11; - uint2 __13; - float4 v__11 = *(float4*)(ol + (i_15 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__13))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__13))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __13; - } - sumsq[0] = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int rv = 0; rv < 16; ++rv) { - sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]); - } - tl::__sync_thread_partial<3, 64>(); - sumsq[0] = tl::AllReduce>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192]))); - rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_17 = 0; i_17 < 2; ++i_17) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)]))); - } - tl::cp_async_commit(); - for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_18 = 0; i_18 < 2; ++i_18) { - *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_3 = 0; vec_3 < 2; ++vec_3) { - float4 __14; - uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4)); - ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]); - ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]); - *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __14; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_19 = 0; i_19 < 2; ++i_19) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)]))); - } - tl::cp_async_commit(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_20 = 0; i_20 < 2; ++i_20) { - *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984)); - for (int vec_4 = 0; vec_4 < 2; ++vec_4) { - float4 __15; - float4 __16; - float4 __17; - uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4)); - ((float2*)(&__17))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]); - ((float2*)(&__17))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]); - float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __16.x = (__17.x*v__14.x); - __16.y = (__17.y*v__14.y); - __16.z = (__17.z*v__14.z); - __16.w = (__17.w*v__14.w); - float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4))); - __15.x = (__16.x*v__15.x); - __15.y = (__16.y*v__15.y); - __15.z = (__16.z*v__15.z); - __15.w = (__16.w*v__15.w); - *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __15; - } - } - #pragma unroll - for (int i_21 = 0; i_21 < 2; ++i_21) { - for (int vec_5 = 0; vec_5 < 2; ++vec_5) { - uint2 __18; - float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__18))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__18))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]); - *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __18; - } - *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0); - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_22 = 0; i_22 < 2; ++i_22) { - *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_6 = 0; vec_6 < 2; ++vec_6) { - float4 __19; - uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4)); - ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]); - ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]); - *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __19; - } - } - #pragma unroll - for (int i_23 = 0; i_23 < 2; ++i_23) { - *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032)); - for (int vec_7 = 0; vec_7 < 2; ++vec_7) { - float4 __20; - float4 __21; - float4 __22; - uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4)); - ((float2*)(&__22))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]); - ((float2*)(&__22))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]); - float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __21.x = (__22.x*v__19.x); - __21.y = (__22.y*v__19.y); - __21.z = (__22.z*v__19.z); - __21.w = (__22.w*v__19.w); - float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4))); - __20.x = (__21.x*v__20.x); - __20.y = (__21.y*v__20.y); - __20.z = (__21.z*v__20.z); - __20.w = (__21.w*v__20.w); - *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __20; - } - } - #pragma unroll - for (int i_24 = 0; i_24 < 2; ++i_24) { - for (int vec_8 = 0; vec_8 < 2; ++vec_8) { - uint2 __23; - float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__23))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__23))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]); - *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __23; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0); - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_25 = 0; i_25 < 2; ++i_25) { - *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104)); - for (int vec_9 = 0; vec_9 < 2; ++vec_9) { - float4 __24; - uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4)); - ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]); - ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]); - *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __24; - } - } - #pragma unroll - for (int i_26 = 0; i_26 < 2; ++i_26) { - *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056)); - for (int vec_10 = 0; vec_10 < 2; ++vec_10) { - float4 __25; - float4 __26; - float4 __27; - uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4)); - ((float2*)(&__27))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]); - ((float2*)(&__27))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]); - float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __26.x = (__27.x*v__24.x); - __26.y = (__27.y*v__24.y); - __26.z = (__27.z*v__24.z); - __26.w = (__27.w*v__24.w); - float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4))); - __25.x = (__26.x*v__25.x); - __25.y = (__26.y*v__25.y); - __25.z = (__26.z*v__25.z); - __25.w = (__26.w*v__25.w); - *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __25; - } - } - #pragma unroll - for (int i_27 = 0; i_27 < 2; ++i_27) { - for (int vec_11 = 0; vec_11 < 2; ++vec_11) { - uint2 __28; - float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__28))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__28))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]); - *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __28; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/690cfe1726579f52e2028c161d3001b9f82af97555a40aab825d6e0140ad4e12/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/690cfe1726579f52e2028c161d3001b9f82af97555a40aab825d6e0140ad4e12/host_kernel.cu deleted file mode 100644 index e72b9ff..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/690cfe1726579f52e2028c161d3001b9f82af97555a40aab825d6e0140ad4e12/host_kernel.cu +++ /dev/null @@ -1,3289 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_pre_big_fuse_with_norm_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[16]; - void* stack_ffi_any = stack; - if (!((num_args == 9))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_pre_big_fuse_with_norm_tilelang: num_args should be 9", (long long)(num_args), (long long)(9)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang: args pointer is NULL"); - return -1; - } - int32_t gemm_out_mul_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((gemm_out_mul_type_index == 0) || (gemm_out_mul_type_index == 4)) || (gemm_out_mul_type_index == 7)) || (64 <= gemm_out_mul_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_mul expected pointer or tensor handle"); - return -1; - } - int32_t gemm_out_sqrsum_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((gemm_out_sqrsum_type_index == 0) || (gemm_out_sqrsum_type_index == 4)) || (gemm_out_sqrsum_type_index == 7)) || (64 <= gemm_out_sqrsum_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_sqrsum expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t layer_input_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((layer_input_type_index == 0) || (layer_input_type_index == 4)) || (layer_input_type_index == 7)) || (64 <= layer_input_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input layer_input expected pointer or tensor handle"); - return -1; - } - int32_t norm_weight_type_index = (((TVMFFIAny*)args)[8].type_index); - if (!(((((norm_weight_type_index == 0) || (norm_weight_type_index == 4)) || (norm_weight_type_index == 7)) || (64 <= norm_weight_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input norm_weight expected pointer or tensor handle"); - return -1; - } - void* gemm_out_mul = ((gemm_out_mul_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* gemm_out_sqrsum = ((gemm_out_sqrsum_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* layer_input = ((layer_input_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - void* norm_weight = ((norm_weight_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[8].v_ptr) + 24)) : (((TVMFFIAny*)args)[8].v_ptr)); - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null = (gemm_out_mul == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null = (gemm_out_sqrsum == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_residual_is_null = (residual == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null = (layer_input == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.layer_input is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null = (norm_weight == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.norm_weight is expected to have non-NULL pointer"); - return -1; - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape = (((DLTensor*)gemm_out_mul)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape = (((DLTensor*)gemm_out_sqrsum)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape = (((DLTensor*)layer_input)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape = (((DLTensor*)norm_weight)[0].shape); - if ((((DLTensor*)gemm_out_mul)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul, mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum, mhc_pre_big_fuse_with_norm_tilelang.residual, mhc_pre_big_fuse_with_norm_tilelang.post_mix, mhc_pre_big_fuse_with_norm_tilelang.comb_mix, mhc_pre_big_fuse_with_norm_tilelang.layer_input"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides = (((DLTensor*)gemm_out_mul)[0].strides); - int32_t dev_id = (((DLTensor*)gemm_out_mul)[0].device.device_id); - void* gemm_out_mul_1 = (((DLTensor*)gemm_out_mul)[0].data); - if ((((DLTensor*)gemm_out_sqrsum)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides = (((DLTensor*)gemm_out_sqrsum)[0].strides); - void* gemm_out_sqrsum_1 = (((DLTensor*)gemm_out_sqrsum)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)comb_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)layer_input)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides = (((DLTensor*)layer_input)[0].strides); - void* layer_input_1 = (((DLTensor*)layer_input)[0].data); - if ((((DLTensor*)norm_weight)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_9) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides = (((DLTensor*)norm_weight)[0].strides); - void* norm_weight_1 = (((DLTensor*)norm_weight)[0].data); - if ((((((DLTensor*)gemm_out_mul)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_mul)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_mul)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0]) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[1]); - } - if ((condval_2 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_mul)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_15) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_mul)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_17) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)gemm_out_sqrsum)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_18) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0]) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - int32_t condval_3; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[1]); - } - if ((condval_3 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_21) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_sqrsum)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_22) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_23) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_25) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_26) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_27) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - if (condval_4 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_5; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_5); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_28) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_29) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_32) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_33) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_34) != 0) { - return -1; - } - } - int32_t condval_6; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - if (condval_6 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_7; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_7); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_35) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_36) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_38) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_39) != 0) { - return -1; - } - } - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_40) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_41) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_42) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - int32_t condval_8; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - if (condval_8 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_9; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_9); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - if (condval_10 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[0]); - } - if ((condval_12 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_46) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_47) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_48) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_49) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_50) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_51) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_52) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_53) != 0) { - return -1; - } - } - int32_t condval_13; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - if (condval_13 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_14; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_14); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_54) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[0]); - } - if ((condval_15 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_55) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_56) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_57) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_58) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_59) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_60) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_61) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1]) != 16) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_62) != 0) { - return -1; - } - } - int32_t condval_16; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - if (condval_16 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_17; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_17); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_63) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[0]); - } - if ((condval_18 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_64) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_65) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_66) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_67) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_68) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)layer_input)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)layer_input)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)layer_input)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_69) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_70) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_71) != 0) { - return -1; - } - } - int32_t condval_19; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - if (condval_19 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_20; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_20); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_72) != 0) { - return -1; - } - } - int32_t condval_21; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[0]); - } - if ((condval_21 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_73) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)layer_input)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_74) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_75) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_76) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_77) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)norm_weight)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)norm_weight)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)norm_weight)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_78) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_79) != 0) { - return -1; - } - } - int32_t condval_22; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - if (condval_22 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_23; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_23); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_80) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)norm_weight)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_81) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_83) != 0) { - return -1; - } - } - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_84) != 0) { - return -1; - } - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_85) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = gemm_out_mul_1; - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = gemm_out_sqrsum_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = hc_scale_1; - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = layer_input_1; - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = norm_weight_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = post_mix_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)96); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)29040); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = (int64_t)0; - if (mhc_pre_big_fuse_with_norm_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_pre_big_fuse_with_norm_tilelang_kernel", &mhc_pre_big_fuse_with_norm_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_pre_big_fuse_with_norm_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 15, &result_86) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/690cfe1726579f52e2028c161d3001b9f82af97555a40aab825d6e0140ad4e12/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/690cfe1726579f52e2028c161d3001b9f82af97555a40aab825d6e0140ad4e12/params.pkl deleted file mode 100644 index e42f0d4cb0923f8fa59c497f5996a89a4f8d2619..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2200 zcmeHI&r2IY6jozmKq~yL9pu`z6nax0Ux zjy70ji**ce3#?OCVO^sWX8UOjeu!kx(Dfs3`B_0f^mKuKr1RFF`s|kvT{RONQFa=%roWn3w3xs@6j4)XjW_gP3cp&yFMXqciV>Mv#{ zK8QwBto`e+*cdsxEp|qxiQQRnYih4(g->fU-F%)}56iXM|2$c%$%qzda+677QWprOvWn@f -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float mixes[1]; - float rms[1]; - float cm[1]; - float row_max[1]; - float row_sum[1]; - float col_sum[1]; - float sumsq_per_pos[16]; - float sumsq[1]; - float rsqrt_norm[1]; - float xl[64]; - float ol[16]; - bfloat16_t xs_local_cast[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t xs_local_cast_2[8]; - float w_local[16]; - float ol_1[16]; - bfloat16_t w_shared_local_cast_3[8]; - bfloat16_t output_shared_local_cast_4[8]; - bfloat16_t layer_input_local_cast_5[8]; - bfloat16_t w_shared_local_cast_6[8]; - bfloat16_t output_shared_local_cast_7[8]; - bfloat16_t layer_input_local_cast_8[8]; - bfloat16_t w_shared_local_cast_9[8]; - bfloat16_t output_shared_local_cast_10[8]; - bfloat16_t layer_input_local_cast_11[8]; - mixes[0] = 0x0p+0f/*0.000000e+00*/; - rms[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - for (int i_split = 0; i_split < 5; ++i_split) { - rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]); - } - rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - mixes[0] = 0x0p+0f/*0.000000e+00*/; - for (int i_split_1 = 0; i_split_1 < 5; ++i_split_1) { - mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]); - } - mixes[0] = (mixes[0] * rms[0]); - if ((((int)threadIdx.x) / 24) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0]; - } - __syncthreads(); - if (((int)threadIdx.x) < 32) { - if (((int)threadIdx.x) < 4) { - post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/); - } - cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]); - row_max[0] = -CUDART_INF_F; - row_max[0] = max(row_max[0], cm[0]); - row_max[0] = tl::AllReduce>::run(row_max[0]); - cm[0] = expf((cm[0] - row_max[0])); - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - for (int __1 = 0; __1 < 19; ++__1) { - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - } - if ((((int)threadIdx.x) >> 4) == 0) { - comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0]; - } - } else { - if (((int)threadIdx.x) < 36) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - #pragma unroll - for (int i = 0; i < 4; ++i) { - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_2 = 0; i_2 < 8; ++i_2) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_4 = 0; i_4 < 8; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - tl::__sync_thread_partial<3, 64>(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)]; - #pragma unroll - for (int i_6 = 0; i_6 < 16; ++i_6) { - ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)])); - } - } - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - float4 __3; - float4 v__1 = *(float4*)(sumsq_per_pos + (i_7 * 4)); - float4 __4; - float4 v__2 = *(float4*)(ol + (i_7 * 4)); - __4.x = (v__2.x*v__2.x); - __4.y = (v__2.y*v__2.y); - __4.z = (v__2.z*v__2.z); - __4.w = (v__2.w*v__2.w); - __3.x = (v__1.x+__4.x); - __3.y = (v__1.y+__4.y); - __3.z = (v__1.z+__4.z); - __3.w = (v__1.w+__4.w); - *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3; - uint2 __5; - float4 v__3 = *(float4*)(ol + (i_7 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __5; - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_8 = 0; i_8 < 8; ++i_8) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __6; - uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]); - *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __6; - } - } - #pragma unroll - for (int i_9 = 0; i_9 < 4; ++i_9) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)]; - #pragma unroll - for (int i_10 = 0; i_10 < 16; ++i_10) { - ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)])); - } - } - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - float4 __7; - float4 v__5 = *(float4*)(sumsq_per_pos + (i_11 * 4)); - float4 __8; - float4 v__6 = *(float4*)(ol + (i_11 * 4)); - __8.x = (v__6.x*v__6.x); - __8.y = (v__6.y*v__6.y); - __8.z = (v__6.z*v__6.z); - __8.w = (v__6.w*v__6.w); - __7.x = (v__5.x+__8.x); - __7.y = (v__5.y+__8.y); - __7.z = (v__5.z+__8.z); - __7.w = (v__5.w+__8.w); - *(float4*)(sumsq_per_pos + (i_11 * 4)) = __7; - uint2 __9; - float4 v__7 = *(float4*)(ol + (i_11 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__9))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__9))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __9; - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_12 = 0; i_12 < 8; ++i_12) { - *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __10; - uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4)); - ((float2*)(&__10))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]); - ((float2*)(&__10))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]); - *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __10; - } - } - #pragma unroll - for (int i_13 = 0; i_13 < 4; ++i_13) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)]; - #pragma unroll - for (int i_14 = 0; i_14 < 16; ++i_14) { - ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)])); - } - } - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - float4 __11; - float4 v__9 = *(float4*)(sumsq_per_pos + (i_15 * 4)); - float4 __12; - float4 v__10 = *(float4*)(ol + (i_15 * 4)); - __12.x = (v__10.x*v__10.x); - __12.y = (v__10.y*v__10.y); - __12.z = (v__10.z*v__10.z); - __12.w = (v__10.w*v__10.w); - __11.x = (v__9.x+__12.x); - __11.y = (v__9.y+__12.y); - __11.z = (v__9.z+__12.z); - __11.w = (v__9.w+__12.w); - *(float4*)(sumsq_per_pos + (i_15 * 4)) = __11; - uint2 __13; - float4 v__11 = *(float4*)(ol + (i_15 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__13))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__13))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __13; - } - sumsq[0] = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int rv = 0; rv < 16; ++rv) { - sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]); - } - tl::__sync_thread_partial<3, 64>(); - sumsq[0] = tl::AllReduce>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192]))); - rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_17 = 0; i_17 < 2; ++i_17) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)]))); - } - tl::cp_async_commit(); - for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_18 = 0; i_18 < 2; ++i_18) { - *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_3 = 0; vec_3 < 2; ++vec_3) { - float4 __14; - uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4)); - ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]); - ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]); - *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __14; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_19 = 0; i_19 < 2; ++i_19) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)]))); - } - tl::cp_async_commit(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_20 = 0; i_20 < 2; ++i_20) { - *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984)); - for (int vec_4 = 0; vec_4 < 2; ++vec_4) { - float4 __15; - float4 __16; - float4 __17; - uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4)); - ((float2*)(&__17))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]); - ((float2*)(&__17))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]); - float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __16.x = (__17.x*v__14.x); - __16.y = (__17.y*v__14.y); - __16.z = (__17.z*v__14.z); - __16.w = (__17.w*v__14.w); - float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4))); - __15.x = (__16.x*v__15.x); - __15.y = (__16.y*v__15.y); - __15.z = (__16.z*v__15.z); - __15.w = (__16.w*v__15.w); - *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __15; - } - } - #pragma unroll - for (int i_21 = 0; i_21 < 2; ++i_21) { - for (int vec_5 = 0; vec_5 < 2; ++vec_5) { - uint2 __18; - float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__18))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__18))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]); - *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __18; - } - *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0); - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_22 = 0; i_22 < 2; ++i_22) { - *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_6 = 0; vec_6 < 2; ++vec_6) { - float4 __19; - uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4)); - ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]); - ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]); - *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __19; - } - } - #pragma unroll - for (int i_23 = 0; i_23 < 2; ++i_23) { - *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032)); - for (int vec_7 = 0; vec_7 < 2; ++vec_7) { - float4 __20; - float4 __21; - float4 __22; - uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4)); - ((float2*)(&__22))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]); - ((float2*)(&__22))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]); - float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __21.x = (__22.x*v__19.x); - __21.y = (__22.y*v__19.y); - __21.z = (__22.z*v__19.z); - __21.w = (__22.w*v__19.w); - float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4))); - __20.x = (__21.x*v__20.x); - __20.y = (__21.y*v__20.y); - __20.z = (__21.z*v__20.z); - __20.w = (__21.w*v__20.w); - *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __20; - } - } - #pragma unroll - for (int i_24 = 0; i_24 < 2; ++i_24) { - for (int vec_8 = 0; vec_8 < 2; ++vec_8) { - uint2 __23; - float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__23))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__23))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]); - *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __23; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0); - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_25 = 0; i_25 < 2; ++i_25) { - *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104)); - for (int vec_9 = 0; vec_9 < 2; ++vec_9) { - float4 __24; - uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4)); - ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]); - ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]); - *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __24; - } - } - #pragma unroll - for (int i_26 = 0; i_26 < 2; ++i_26) { - *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056)); - for (int vec_10 = 0; vec_10 < 2; ++vec_10) { - float4 __25; - float4 __26; - float4 __27; - uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4)); - ((float2*)(&__27))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]); - ((float2*)(&__27))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]); - float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __26.x = (__27.x*v__24.x); - __26.y = (__27.y*v__24.y); - __26.z = (__27.z*v__24.z); - __26.w = (__27.w*v__24.w); - float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4))); - __25.x = (__26.x*v__25.x); - __25.y = (__26.y*v__25.y); - __25.z = (__26.z*v__25.z); - __25.w = (__26.w*v__25.w); - *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __25; - } - } - #pragma unroll - for (int i_27 = 0; i_27 < 2; ++i_27) { - for (int vec_11 = 0; vec_11 < 2; ++vec_11) { - uint2 __28; - float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__28))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__28))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]); - *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __28; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/8ac26e5a6bf2572ea507280b0cfe38c49999af7cb683ccecde6888c37a6e6f3a/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/8ac26e5a6bf2572ea507280b0cfe38c49999af7cb683ccecde6888c37a6e6f3a/host_kernel.cu deleted file mode 100644 index 1530b54..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/8ac26e5a6bf2572ea507280b0cfe38c49999af7cb683ccecde6888c37a6e6f3a/host_kernel.cu +++ /dev/null @@ -1,3359 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_pre_big_fuse_with_norm_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[16]; - void* stack_ffi_any = stack; - if (!((num_args == 9))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_pre_big_fuse_with_norm_tilelang: num_args should be 9", (long long)(num_args), (long long)(9)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang: args pointer is NULL"); - return -1; - } - int32_t gemm_out_mul_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((gemm_out_mul_type_index == 0) || (gemm_out_mul_type_index == 4)) || (gemm_out_mul_type_index == 7)) || (64 <= gemm_out_mul_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_mul expected pointer or tensor handle"); - return -1; - } - int32_t gemm_out_sqrsum_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((gemm_out_sqrsum_type_index == 0) || (gemm_out_sqrsum_type_index == 4)) || (gemm_out_sqrsum_type_index == 7)) || (64 <= gemm_out_sqrsum_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_sqrsum expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t layer_input_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((layer_input_type_index == 0) || (layer_input_type_index == 4)) || (layer_input_type_index == 7)) || (64 <= layer_input_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input layer_input expected pointer or tensor handle"); - return -1; - } - int32_t norm_weight_type_index = (((TVMFFIAny*)args)[8].type_index); - if (!(((((norm_weight_type_index == 0) || (norm_weight_type_index == 4)) || (norm_weight_type_index == 7)) || (64 <= norm_weight_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input norm_weight expected pointer or tensor handle"); - return -1; - } - void* gemm_out_mul = ((gemm_out_mul_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* gemm_out_sqrsum = ((gemm_out_sqrsum_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* layer_input = ((layer_input_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - void* norm_weight = ((norm_weight_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[8].v_ptr) + 24)) : (((TVMFFIAny*)args)[8].v_ptr)); - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null = (gemm_out_mul == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null = (gemm_out_sqrsum == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_residual_is_null = (residual == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null = (layer_input == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.layer_input is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null = (norm_weight == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.norm_weight is expected to have non-NULL pointer"); - return -1; - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape = (((DLTensor*)gemm_out_mul)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape = (((DLTensor*)gemm_out_sqrsum)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape = (((DLTensor*)layer_input)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape = (((DLTensor*)norm_weight)[0].shape); - if ((((DLTensor*)gemm_out_mul)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul, mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum, mhc_pre_big_fuse_with_norm_tilelang.residual, mhc_pre_big_fuse_with_norm_tilelang.post_mix, mhc_pre_big_fuse_with_norm_tilelang.comb_mix, mhc_pre_big_fuse_with_norm_tilelang.layer_input"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides = (((DLTensor*)gemm_out_mul)[0].strides); - int32_t dev_id = (((DLTensor*)gemm_out_mul)[0].device.device_id); - void* gemm_out_mul_1 = (((DLTensor*)gemm_out_mul)[0].data); - if ((((DLTensor*)gemm_out_sqrsum)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides = (((DLTensor*)gemm_out_sqrsum)[0].strides); - void* gemm_out_sqrsum_1 = (((DLTensor*)gemm_out_sqrsum)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)comb_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)layer_input)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides = (((DLTensor*)layer_input)[0].strides); - void* layer_input_1 = (((DLTensor*)layer_input)[0].data); - if ((((DLTensor*)norm_weight)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_9) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides = (((DLTensor*)norm_weight)[0].strides); - void* norm_weight_1 = (((DLTensor*)norm_weight)[0].data); - if ((((((DLTensor*)gemm_out_mul)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_mul)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_mul)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0]) != 5) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)5; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[1]); - } - if ((condval_2 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - int32_t condval_3; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[0]); - } - if (((num_tokens * 24) != condval_3) && (num_tokens != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_15) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_mul)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_mul)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_17) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_18) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)gemm_out_sqrsum)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0]) != 5) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)5; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[1]); - } - if ((condval_4 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_22) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[0]); - } - if ((((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != condval_5) && (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_23) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_sqrsum)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_26) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_27) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_28) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_29) != 0) { - return -1; - } - } - int32_t condval_6; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - if (condval_6 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_7; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_7); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_34) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_35) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_36) != 0) { - return -1; - } - } - int32_t condval_8; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - if (condval_8 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_9; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_9); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_38) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_40) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_41) != 0) { - return -1; - } - } - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_42) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_46) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - if (condval_12 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_13; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_13); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_47) != 0) { - return -1; - } - } - int32_t condval_14; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[0]); - } - if ((condval_14 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_48) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_49) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_51) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_52) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_53) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_54) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - if (condval_15 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_16; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_16); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_56) != 0) { - return -1; - } - } - int32_t condval_17; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[0]); - } - if ((condval_17 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_58) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_60) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_61) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_62) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_63) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1]) != 16) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_64) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - if (condval_18 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[0]); - } - if ((condval_20 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_66) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_67) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_68) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_69) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_70) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)layer_input)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)layer_input)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)layer_input)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_71) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_72) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_73) != 0) { - return -1; - } - } - int32_t condval_21; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - if (condval_21 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_22; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_22); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - int32_t condval_23; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[0]); - } - if ((condval_23 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_75) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)layer_input)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_76) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_77) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_78) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_79) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)norm_weight)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)norm_weight)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)norm_weight)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_80) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - if (condval_24 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_25; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_25); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)norm_weight)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_83) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_85) != 0) { - return -1; - } - } - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_86) != 0) { - return -1; - } - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_87) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = gemm_out_mul_1; - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = gemm_out_sqrsum_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = hc_scale_1; - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = layer_input_1; - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = norm_weight_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = post_mix_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)96); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)29040); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = (int64_t)0; - if (mhc_pre_big_fuse_with_norm_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_pre_big_fuse_with_norm_tilelang_kernel", &mhc_pre_big_fuse_with_norm_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_pre_big_fuse_with_norm_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 15, &result_88) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/8ac26e5a6bf2572ea507280b0cfe38c49999af7cb683ccecde6888c37a6e6f3a/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/8ac26e5a6bf2572ea507280b0cfe38c49999af7cb683ccecde6888c37a6e6f3a/params.pkl deleted file mode 100644 index cef5ff9ceabecd0481845936e008448d33e9c5af..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2200 zcmeHI&r2IY6jozmKq~<$YN24~6d^4BfF5%8HdMrlmm+IQB!U zV%04(1wWY%&!J-xTowwfGyzmz4BiO(a`#FtrcYih4(g->fU-FTi_56iXM{W@8z$%qzd<8cMb5s4Wa t5MBH>*O+=r&Dpc--K*kyebY_(;=g*|!{Uv4^;vmk3*YV*CgYU{!C!ev_6q<2 diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/94b96c76ed4b18c5a1404f5c71d4cc9c8be8d3d582285c754714502150b21353/device_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/94b96c76ed4b18c5a1404f5c71d4cc9c8be8d3d582285c754714502150b21353/device_kernel.cu deleted file mode 100644 index 7c69186..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/94b96c76ed4b18c5a1404f5c71d4cc9c8be8d3d582285c754714502150b21353/device_kernel.cu +++ /dev/null @@ -1,104 +0,0 @@ -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_fused_tilelang_kernel(const float* comb_mix, const float* post_mix, const bfloat16_t* residual_in, bfloat16_t* residual_out, float* rp_out, const float* weight_t, const bfloat16_t* x_in, float* yp_out, int num_tokens, int split_k); -extern "C" __global__ void __launch_bounds__(256, 1) mhc_fused_tilelang_kernel(const float* comb_mix, const float* post_mix, const bfloat16_t* residual_in, bfloat16_t* residual_out, float* rp_out, const float* weight_t, const bfloat16_t* x_in, float* yp_out, int num_tokens, int split_k) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float acc[3]; - float sqr[1]; - float pm[4]; - float cm[16]; - float new_r[4]; - float v = 0x0p+0f/*0.000000e+00*/; - float v2 = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int i = 0; i < 3; ++i) { - acc[i] = 0x0p+0f/*0.000000e+00*/; - } - sqr[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - if (((int)threadIdx.x) < 4) { - ((float*)buf_dyn_shmem)[((int)threadIdx.x)] = post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))]; - } - if (((int)threadIdx.x) < 16) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] = comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + ((int64_t)((int)threadIdx.x)))]; - } - __syncthreads(); - #pragma unroll - for (int j = 0; j < 4; ++j) { - pm[j] = ((float*)buf_dyn_shmem)[j]; - } - #pragma unroll - for (int j_1 = 0; j_1 < 4; ++j_1) { - #pragma unroll - for (int k = 0; k < 4; ++k) { - cm[((k * 4) + j_1)] = ((float*)buf_dyn_shmem)[(((k * 4) + j_1) + 4)]; - } - } - for (int it = 0; it < ((4096 / split_k) >> 8); ++it) { - #pragma unroll - for (int j_2 = 0; j_2 < 4; ++j_2) { - new_r[j_2] = (pm[j_2] * ((float)x_in[((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)it) * (int64_t)256)) + (((int64_t)((int)blockIdx.z)) * ((int64_t)4096 / ((int64_t)split_k)))) + ((int64_t)((int)threadIdx.x)))])); - #pragma unroll - for (int k_1 = 0; k_1 < 4; ++k_1) { - new_r[j_2] = (new_r[j_2] + (cm[((k_1 * 4) + j_2)] * ((float)residual_in[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + (((int64_t)k_1) * (int64_t)4096)) + (((int64_t)it) * (int64_t)256)) + (((int64_t)((int)blockIdx.z)) * ((int64_t)4096 / ((int64_t)split_k)))) + ((int64_t)((int)threadIdx.x)))]))); - } - } - if (((int)blockIdx.y) == 0) { - #pragma unroll - for (int j_3 = 0; j_3 < 4; ++j_3) { - residual_out[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + (((int64_t)j_3) * (int64_t)4096)) + (((int64_t)it) * (int64_t)256)) + (((int64_t)((int)blockIdx.z)) * ((int64_t)4096 / ((int64_t)split_k)))) + ((int64_t)((int)threadIdx.x)))] = ((bfloat16_t)new_r[j_3]); - sqr[0] = (sqr[0] + (new_r[j_3] * new_r[j_3])); - } - } - #pragma unroll - for (int n = 0; n < 3; ++n) { - #pragma unroll - for (int j_4 = 0; j_4 < 4; ++j_4) { - acc[n] = (acc[n] + (weight_t[((((((((int64_t)((int)blockIdx.y)) * (int64_t)49152) + (((int64_t)n) * (int64_t)16384)) + (((int64_t)j_4) * (int64_t)4096)) + (((int64_t)it) * (int64_t)256)) + (((int64_t)((int)blockIdx.z)) * ((int64_t)4096 / ((int64_t)split_k)))) + ((int64_t)((int)threadIdx.x)))] * new_r[j_4])); - } - } - } - #pragma unroll - for (int n_1 = 0; n_1 < 3; ++n_1) { - acc[n_1] = tl::warp_reduce_sum(acc[n_1]); - } - if (((int)blockIdx.y) == 0) { - sqr[0] = tl::warp_reduce_sum(sqr[0]); - } - if ((((int)threadIdx.x) % 32) == 0) { - #pragma unroll - for (int n_2 = 0; n_2 < 3; ++n_2) { - ((float*)buf_dyn_shmem)[((((((int)threadIdx.x) >> 5) * 4) + n_2) + 20)] = acc[n_2]; - } - if (((int)blockIdx.y) == 0) { - ((float*)buf_dyn_shmem)[(((((int)threadIdx.x) >> 5) * 4) + 23)] = sqr[0]; - } - } - __syncthreads(); - if ((((int)threadIdx.x) >> 5) == 0) { - if ((((int)threadIdx.x) & 31) < 3) { - #pragma unroll - for (int w = 0; w < 8; ++w) { - v = (v + ((float*)buf_dyn_shmem)[(((w * 4) + (((int)threadIdx.x) & 31)) + 20)]); - } - yp_out[((((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)((int)blockIdx.z)) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)blockIdx.y)) * (int64_t)3)) + (((int64_t)((int)threadIdx.x)) & (int64_t)31))] = v; - } - if ((((int)blockIdx.y) == 0) && ((((int)threadIdx.x) % 32) == 0)) { - #pragma unroll - for (int w_1 = 0; w_1 < 8; ++w_1) { - v2 = (v2 + ((float*)buf_dyn_shmem)[((w_1 * 4) + 23)]); - } - rp_out[((((int64_t)((int)blockIdx.z)) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))] = v2; - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/94b96c76ed4b18c5a1404f5c71d4cc9c8be8d3d582285c754714502150b21353/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/94b96c76ed4b18c5a1404f5c71d4cc9c8be8d3d582285c754714502150b21353/host_kernel.cu deleted file mode 100644 index 8167a97..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/94b96c76ed4b18c5a1404f5c71d4cc9c8be8d3d582285c754714502150b21353/host_kernel.cu +++ /dev/null @@ -1,3435 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_fused_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_fused_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_fused_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[18]; - void* stack_ffi_any = stack; - if (!((num_args == 8))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_fused_tilelang: num_args should be 8", (long long)(num_args), (long long)(8)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang: args pointer is NULL"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_fused_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t residual_in_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((residual_in_type_index == 0) || (residual_in_type_index == 4)) || (residual_in_type_index == 7)) || (64 <= residual_in_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_fused_tilelang input residual_in expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_fused_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t x_in_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((x_in_type_index == 0) || (x_in_type_index == 4)) || (x_in_type_index == 7)) || (64 <= x_in_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_fused_tilelang input x_in expected pointer or tensor handle"); - return -1; - } - int32_t weight_t_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((weight_t_type_index == 0) || (weight_t_type_index == 4)) || (weight_t_type_index == 7)) || (64 <= weight_t_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_fused_tilelang input weight_t expected pointer or tensor handle"); - return -1; - } - int32_t yp_out_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((yp_out_type_index == 0) || (yp_out_type_index == 4)) || (yp_out_type_index == 7)) || (64 <= yp_out_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_fused_tilelang input yp_out expected pointer or tensor handle"); - return -1; - } - int32_t rp_out_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((rp_out_type_index == 0) || (rp_out_type_index == 4)) || (rp_out_type_index == 7)) || (64 <= rp_out_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_fused_tilelang input rp_out expected pointer or tensor handle"); - return -1; - } - int32_t residual_out_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((residual_out_type_index == 0) || (residual_out_type_index == 4)) || (residual_out_type_index == 7)) || (64 <= residual_out_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_fused_tilelang input residual_out expected pointer or tensor handle"); - return -1; - } - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* residual_in = ((residual_in_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* x_in = ((x_in_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* weight_t = ((weight_t_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* yp_out = ((yp_out_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* rp_out = ((rp_out_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* residual_out = ((residual_out_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - bool mhc_fused_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_fused_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_fused_tilelang_residual_in_is_null = (residual_in == NULL); - if (!(!mhc_fused_tilelang_residual_in_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang.residual_in is expected to have non-NULL pointer"); - return -1; - } - bool mhc_fused_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_fused_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_fused_tilelang_x_in_is_null = (x_in == NULL); - if (!(!mhc_fused_tilelang_x_in_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang.x_in is expected to have non-NULL pointer"); - return -1; - } - bool mhc_fused_tilelang_weight_t_is_null = (weight_t == NULL); - if (!(!mhc_fused_tilelang_weight_t_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang.weight_t is expected to have non-NULL pointer"); - return -1; - } - bool mhc_fused_tilelang_yp_out_is_null = (yp_out == NULL); - if (!(!mhc_fused_tilelang_yp_out_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang.yp_out is expected to have non-NULL pointer"); - return -1; - } - bool mhc_fused_tilelang_rp_out_is_null = (rp_out == NULL); - if (!(!mhc_fused_tilelang_rp_out_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang.rp_out is expected to have non-NULL pointer"); - return -1; - } - bool mhc_fused_tilelang_residual_out_is_null = (residual_out == NULL); - if (!(!mhc_fused_tilelang_residual_out_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang.residual_out is expected to have non-NULL pointer"); - return -1; - } - void* mhc_fused_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_fused_tilelang_residual_in_shape = (((DLTensor*)residual_in)[0].shape); - void* mhc_fused_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_fused_tilelang_x_in_shape = (((DLTensor*)x_in)[0].shape); - void* mhc_fused_tilelang_weight_t_shape = (((DLTensor*)weight_t)[0].shape); - void* mhc_fused_tilelang_yp_out_shape = (((DLTensor*)yp_out)[0].shape); - void* mhc_fused_tilelang_rp_out_shape = (((DLTensor*)rp_out)[0].shape); - void* mhc_fused_tilelang_residual_out_shape = (((DLTensor*)residual_out)[0].shape); - if ((((DLTensor*)comb_mix)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_fused_tilelang.comb_mix, mhc_fused_tilelang.residual_in, mhc_fused_tilelang.post_mix, mhc_fused_tilelang.x_in, mhc_fused_tilelang.yp_out, mhc_fused_tilelang.rp_out, mhc_fused_tilelang.residual_out"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]); - void* mhc_fused_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - int32_t dev_id = (((DLTensor*)comb_mix)[0].device.device_id); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)residual_in)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual_in)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_fused_tilelang_residual_in_strides = (((DLTensor*)residual_in)[0].strides); - void* residual_in_1 = (((DLTensor*)residual_in)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_fused_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)x_in)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)x_in)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_fused_tilelang_x_in_strides = (((DLTensor*)x_in)[0].strides); - void* x_in_1 = (((DLTensor*)x_in)[0].data); - if ((((DLTensor*)weight_t)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)weight_t)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_fused_tilelang_weight_t_strides = (((DLTensor*)weight_t)[0].strides); - void* weight_t_1 = (((DLTensor*)weight_t)[0].data); - if ((((DLTensor*)yp_out)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)yp_out)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable split_k requires at least one non-null buffer among: mhc_fused_tilelang.yp_out, mhc_fused_tilelang.rp_out"); - return -1; - } - int32_t split_k = ((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_shape)[0]); - void* mhc_fused_tilelang_yp_out_strides = (((DLTensor*)yp_out)[0].strides); - void* yp_out_1 = (((DLTensor*)yp_out)[0].data); - if ((((DLTensor*)rp_out)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)rp_out)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_fused_tilelang_rp_out_strides = (((DLTensor*)rp_out)[0].strides); - void* rp_out_1 = (((DLTensor*)rp_out)[0].data); - if ((((DLTensor*)residual_out)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual_out)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_fused_tilelang_residual_out_strides = (((DLTensor*)residual_out)[0].strides); - void* residual_out_1 = (((DLTensor*)residual_out)[0].data); - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_9) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[2]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_fused_tilelang_comb_mix_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_fused_tilelang_comb_mix_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_fused_tilelang_comb_mix_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_strides)[1]); - } - if (condval_2 != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_3; - if ((mhc_fused_tilelang_comb_mix_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_3); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_fused_tilelang_comb_mix_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_strides)[0]); - } - if ((condval_4 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_15) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_17) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)residual_in)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual_in)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual_in)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual_in)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual_in)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual_in)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_18) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]) != ((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_fused_tilelang_residual_in_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_strides)[2]); - } - if (condval_5 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_6; - if ((mhc_fused_tilelang_residual_in_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_6); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_22) != 0) { - return -1; - } - } - int32_t condval_7; - if ((mhc_fused_tilelang_residual_in_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_strides)[1]); - } - if (condval_7 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_8; - if ((mhc_fused_tilelang_residual_in_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_8); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_23) != 0) { - return -1; - } - } - int32_t condval_9; - if ((mhc_fused_tilelang_residual_in_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_strides)[0]); - } - if ((condval_9 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_24) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual_in)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual_in)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)residual_in)[0].device.device_id) != (((DLTensor*)comb_mix)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual_in)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_26) != 0) { - return -1; - } - } - if ((((DLTensor*)residual_in)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual_in)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_27) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_in_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_28) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_29) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]) != ((int32_t)((int64_t*)mhc_fused_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_31) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_fused_tilelang_post_mix_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_fused_tilelang_post_mix_strides)[1]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_fused_tilelang_post_mix_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_fused_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_fused_tilelang_post_mix_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_fused_tilelang_post_mix_strides)[0]); - } - if ((condval_12 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_33) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_34) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)comb_mix)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_35) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_36) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_37) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)x_in)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)x_in)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)x_in)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)x_in)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)x_in)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)x_in)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_38) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]) != ((int32_t)((int64_t*)mhc_fused_tilelang_x_in_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_x_in_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_x_in_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_x_in_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_40) != 0) { - return -1; - } - } - int32_t condval_13; - if ((mhc_fused_tilelang_x_in_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_fused_tilelang_x_in_strides)[1]); - } - if (condval_13 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_14; - if ((mhc_fused_tilelang_x_in_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_fused_tilelang_x_in_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_14); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_41) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_fused_tilelang_x_in_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_fused_tilelang_x_in_strides)[0]); - } - if ((condval_15 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_42) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)x_in)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)x_in)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_43) != 0) { - return -1; - } - } - if ((((DLTensor*)x_in)[0].device.device_id) != (((DLTensor*)comb_mix)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)x_in)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if ((((DLTensor*)x_in)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)x_in)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_45) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (x_in_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_46) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)weight_t)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)weight_t)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)weight_t)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)weight_t)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)weight_t)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)weight_t)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_47) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_48) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_49) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - int32_t condval_16; - if ((mhc_fused_tilelang_weight_t_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_strides)[2]); - } - if (condval_16 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_17; - if ((mhc_fused_tilelang_weight_t_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_17); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_51) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_fused_tilelang_weight_t_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_strides)[1]); - } - if (condval_18 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_fused_tilelang_weight_t_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_52) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_fused_tilelang_weight_t_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_strides)[0]); - } - if (condval_20 != 16384) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16384; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_21; - if ((mhc_fused_tilelang_weight_t_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_21); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_53) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)weight_t)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)weight_t)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_54) != 0) { - return -1; - } - } - if ((((DLTensor*)weight_t)[0].device.device_id) != (((DLTensor*)comb_mix)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)weight_t)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - if ((((DLTensor*)weight_t)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)weight_t)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_56) != 0) { - return -1; - } - } - if (weight_t_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((((((DLTensor*)yp_out)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)yp_out)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)yp_out)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)yp_out)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)yp_out)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)yp_out)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_58) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]) != ((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_60) != 0) { - return -1; - } - } - int32_t condval_22; - if ((mhc_fused_tilelang_yp_out_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_strides)[2]); - } - if (condval_22 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_23; - if ((mhc_fused_tilelang_yp_out_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_23); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_61) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_fused_tilelang_yp_out_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_strides)[1]); - } - if ((condval_24 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_62) != 0) { - return -1; - } - } - int32_t condval_25; - if ((mhc_fused_tilelang_yp_out_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_strides)[0]); - } - if ((((num_tokens * 24) != condval_25) && (num_tokens != 0)) && (split_k != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_63) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)yp_out)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)yp_out)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_64) != 0) { - return -1; - } - } - if ((((DLTensor*)yp_out)[0].device.device_id) != (((DLTensor*)comb_mix)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)yp_out)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - if ((((DLTensor*)yp_out)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)yp_out)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_66) != 0) { - return -1; - } - } - if ((split_k != 0) && (num_tokens != 0)) { - if (yp_out_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_67) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)rp_out)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)rp_out)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)rp_out)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)rp_out)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)rp_out)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)rp_out)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_68) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_shape)[0]) != ((int32_t)((int64_t*)mhc_fused_tilelang_rp_out_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_rp_out_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_69) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]) != ((int32_t)((int64_t*)mhc_fused_tilelang_rp_out_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_rp_out_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_70) != 0) { - return -1; - } - } - int32_t condval_26; - if ((mhc_fused_tilelang_rp_out_strides == NULL)) { - condval_26 = 1; - } else { - condval_26 = ((int32_t)((int64_t*)mhc_fused_tilelang_rp_out_strides)[1]); - } - if ((condval_26 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_71) != 0) { - return -1; - } - } - int32_t condval_27; - if ((mhc_fused_tilelang_rp_out_strides == NULL)) { - condval_27 = 1; - } else { - condval_27 = ((int32_t)((int64_t*)mhc_fused_tilelang_rp_out_strides)[0]); - } - if (((((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]) != condval_27) && (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]) != 0)) && (split_k != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_72) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)rp_out)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)rp_out)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_73) != 0) { - return -1; - } - } - if ((((DLTensor*)rp_out)[0].device.device_id) != (((DLTensor*)comb_mix)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)rp_out)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - if ((((DLTensor*)rp_out)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)rp_out)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_75) != 0) { - return -1; - } - } - if ((split_k != 0) && (num_tokens != 0)) { - if (rp_out_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_76) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)residual_out)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual_out)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual_out)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual_out)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual_out)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual_out)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_77) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]) != ((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_78) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_79) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_80) != 0) { - return -1; - } - } - int32_t condval_28; - if ((mhc_fused_tilelang_residual_out_strides == NULL)) { - condval_28 = 1; - } else { - condval_28 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_strides)[2]); - } - if (condval_28 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_29; - if ((mhc_fused_tilelang_residual_out_strides == NULL)) { - condval_29 = 1; - } else { - condval_29 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_29); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_30; - if ((mhc_fused_tilelang_residual_out_strides == NULL)) { - condval_30 = 1; - } else { - condval_30 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_strides)[1]); - } - if (condval_30 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_31; - if ((mhc_fused_tilelang_residual_out_strides == NULL)) { - condval_31 = 1; - } else { - condval_31 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_31); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - int32_t condval_32; - if ((mhc_fused_tilelang_residual_out_strides == NULL)) { - condval_32 = 1; - } else { - condval_32 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_strides)[0]); - } - if ((condval_32 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_83) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual_out)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual_out)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)residual_out)[0].device.device_id) != (((DLTensor*)comb_mix)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual_out)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_85) != 0) { - return -1; - } - } - if ((((DLTensor*)residual_out)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual_out)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_86) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_out_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_87) != 0) { - return -1; - } - } - } else { - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_88) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = post_mix_1; - if (residual_in_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = residual_in_1; - if (residual_out_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = residual_out_1; - if (rp_out_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = rp_out_1; - if (weight_t_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = weight_t_1; - if (x_in_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = x_in_1; - if (yp_out_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = yp_out_1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)split_k); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)8); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)split_k); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)256); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[16].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[16].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[16].v_int64) = ((int64_t)208); - (((TVMFFIAny*)stack_ffi_any)[17].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[17].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[17].v_int64) = (int64_t)0; - if (mhc_fused_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_fused_tilelang_kernel", &mhc_fused_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_89; - result_89.type_index = kTVMFFINone; - result_89.zero_padding = 0; - result_89.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_fused_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 17, &result_89) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_fused_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/94b96c76ed4b18c5a1404f5c71d4cc9c8be8d3d582285c754714502150b21353/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/94b96c76ed4b18c5a1404f5c71d4cc9c8be8d3d582285c754714502150b21353/params.pkl deleted file mode 100644 index 2a45bcde1473ac97d9936a6f472c0d5b315031a0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2862 zcmeHIL2DC16pn2Y1F<1cN~<8~cxfRlO$#1!_O@7xiWe!vZ1yEH&F;)HJE<*U!5->q zZW;fb;9v3WCaKM$2-bs`>}`hcz4yIu-uw1b?aPa1rTEU=zTGh*f{4kX2XZJRcnQ-i zcDCt*mJpp5r>Q%*b{V2;p}9)H*_OG8>0lr{I=kTYsk09Ukz(ff$lbVWcakBB&S7(u zeY6PcD~(wm)4m2~fcA?lbf8sC%{Ym`O|e)B-FU?edzY7=I)B4|=07m*jY~hfz}8aE z@E-f&E!(6LK2jB&L*F2@NaGoyJ-AG?vzzak#_Jo5MT&LKWQg_bw+Yco87gE5m!x;x zAyS2qlHP|2$$KU}a-t-VtaE$L`+sY+kVDcTA(BIk9AgJ+1Twvg#Nww~#5ws}=!B#R zlcaZuvGW2#qIx8|QzEC0e19K=bXklfyPbk9Wl}1`j8ItND9g?=mj-^-ufjOso3qE{ zb>6Uzpm?>-G_sV -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_fused_tilelang_kernel(const float* comb_mix, const float* post_mix, const bfloat16_t* residual_in, bfloat16_t* residual_out, float* rp_out, const float* weight_t, const bfloat16_t* x_in, float* yp_out, int num_tokens, int split_k); -extern "C" __global__ void __launch_bounds__(256, 1) mhc_fused_tilelang_kernel(const float* comb_mix, const float* post_mix, const bfloat16_t* residual_in, bfloat16_t* residual_out, float* rp_out, const float* weight_t, const bfloat16_t* x_in, float* yp_out, int num_tokens, int split_k) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float acc[2]; - float sqr[1]; - float pm[4]; - float cm[16]; - float new_r[4]; - float v = 0x0p+0f/*0.000000e+00*/; - float v2 = 0x0p+0f/*0.000000e+00*/; - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float2*)(acc + 0) = make_float2(broadcast_var, broadcast_var); - sqr[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - if (((int)threadIdx.x) < 4) { - ((float*)buf_dyn_shmem)[((int)threadIdx.x)] = post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))]; - } - if (((int)threadIdx.x) < 16) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] = comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + ((int64_t)((int)threadIdx.x)))]; - } - __syncthreads(); - #pragma unroll - for (int j = 0; j < 4; ++j) { - pm[j] = ((float*)buf_dyn_shmem)[j]; - } - #pragma unroll - for (int j_1 = 0; j_1 < 4; ++j_1) { - #pragma unroll - for (int k = 0; k < 4; ++k) { - cm[((k * 4) + j_1)] = ((float*)buf_dyn_shmem)[(((k * 4) + j_1) + 4)]; - } - } - for (int it = 0; it < ((4096 / split_k) >> 8); ++it) { - #pragma unroll - for (int j_2 = 0; j_2 < 4; ++j_2) { - new_r[j_2] = (pm[j_2] * ((float)x_in[((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)it) * (int64_t)256)) + (((int64_t)((int)blockIdx.z)) * ((int64_t)4096 / ((int64_t)split_k)))) + ((int64_t)((int)threadIdx.x)))])); - #pragma unroll - for (int k_1 = 0; k_1 < 4; ++k_1) { - new_r[j_2] = (new_r[j_2] + (cm[((k_1 * 4) + j_2)] * ((float)residual_in[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + (((int64_t)k_1) * (int64_t)4096)) + (((int64_t)it) * (int64_t)256)) + (((int64_t)((int)blockIdx.z)) * ((int64_t)4096 / ((int64_t)split_k)))) + ((int64_t)((int)threadIdx.x)))]))); - } - } - if (((int)blockIdx.y) == 0) { - #pragma unroll - for (int j_3 = 0; j_3 < 4; ++j_3) { - residual_out[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + (((int64_t)j_3) * (int64_t)4096)) + (((int64_t)it) * (int64_t)256)) + (((int64_t)((int)blockIdx.z)) * ((int64_t)4096 / ((int64_t)split_k)))) + ((int64_t)((int)threadIdx.x)))] = ((bfloat16_t)new_r[j_3]); - sqr[0] = (sqr[0] + (new_r[j_3] * new_r[j_3])); - } - } - #pragma unroll - for (int n = 0; n < 2; ++n) { - #pragma unroll - for (int j_4 = 0; j_4 < 4; ++j_4) { - acc[n] = (acc[n] + (weight_t[((((((((int64_t)((int)blockIdx.y)) * (int64_t)32768) + (((int64_t)n) * (int64_t)16384)) + (((int64_t)j_4) * (int64_t)4096)) + (((int64_t)it) * (int64_t)256)) + (((int64_t)((int)blockIdx.z)) * ((int64_t)4096 / ((int64_t)split_k)))) + ((int64_t)((int)threadIdx.x)))] * new_r[j_4])); - } - } - } - #pragma unroll - for (int n_1 = 0; n_1 < 2; ++n_1) { - acc[n_1] = tl::warp_reduce_sum(acc[n_1]); - } - if (((int)blockIdx.y) == 0) { - sqr[0] = tl::warp_reduce_sum(sqr[0]); - } - if ((((int)threadIdx.x) % 32) == 0) { - #pragma unroll - for (int n_2 = 0; n_2 < 2; ++n_2) { - ((float*)buf_dyn_shmem)[((((((int)threadIdx.x) >> 5) * 3) + n_2) + 20)] = acc[n_2]; - } - if (((int)blockIdx.y) == 0) { - ((float*)buf_dyn_shmem)[(((((int)threadIdx.x) >> 5) * 3) + 22)] = sqr[0]; - } - } - __syncthreads(); - if ((((int)threadIdx.x) >> 5) == 0) { - if ((((int)threadIdx.x) & 31) < 2) { - #pragma unroll - for (int w = 0; w < 8; ++w) { - v = (v + ((float*)buf_dyn_shmem)[(((w * 3) + (((int)threadIdx.x) & 31)) + 20)]); - } - yp_out[((((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)((int)blockIdx.z)) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)blockIdx.y)) * (int64_t)2)) + (((int64_t)((int)threadIdx.x)) & (int64_t)31))] = v; - } - if ((((int)blockIdx.y) == 0) && ((((int)threadIdx.x) % 32) == 0)) { - #pragma unroll - for (int w_1 = 0; w_1 < 8; ++w_1) { - v2 = (v2 + ((float*)buf_dyn_shmem)[((w_1 * 3) + 22)]); - } - rp_out[((((int64_t)((int)blockIdx.z)) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))] = v2; - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/96e9b9b5cf258ab70fb96c06c4ba12fa70819bfb1b8c147e903abf420053524b/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/96e9b9b5cf258ab70fb96c06c4ba12fa70819bfb1b8c147e903abf420053524b/host_kernel.cu deleted file mode 100644 index 0a225e6..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/96e9b9b5cf258ab70fb96c06c4ba12fa70819bfb1b8c147e903abf420053524b/host_kernel.cu +++ /dev/null @@ -1,3435 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_fused_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_fused_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_fused_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[18]; - void* stack_ffi_any = stack; - if (!((num_args == 8))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_fused_tilelang: num_args should be 8", (long long)(num_args), (long long)(8)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang: args pointer is NULL"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_fused_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t residual_in_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((residual_in_type_index == 0) || (residual_in_type_index == 4)) || (residual_in_type_index == 7)) || (64 <= residual_in_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_fused_tilelang input residual_in expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_fused_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t x_in_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((x_in_type_index == 0) || (x_in_type_index == 4)) || (x_in_type_index == 7)) || (64 <= x_in_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_fused_tilelang input x_in expected pointer or tensor handle"); - return -1; - } - int32_t weight_t_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((weight_t_type_index == 0) || (weight_t_type_index == 4)) || (weight_t_type_index == 7)) || (64 <= weight_t_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_fused_tilelang input weight_t expected pointer or tensor handle"); - return -1; - } - int32_t yp_out_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((yp_out_type_index == 0) || (yp_out_type_index == 4)) || (yp_out_type_index == 7)) || (64 <= yp_out_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_fused_tilelang input yp_out expected pointer or tensor handle"); - return -1; - } - int32_t rp_out_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((rp_out_type_index == 0) || (rp_out_type_index == 4)) || (rp_out_type_index == 7)) || (64 <= rp_out_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_fused_tilelang input rp_out expected pointer or tensor handle"); - return -1; - } - int32_t residual_out_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((residual_out_type_index == 0) || (residual_out_type_index == 4)) || (residual_out_type_index == 7)) || (64 <= residual_out_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_fused_tilelang input residual_out expected pointer or tensor handle"); - return -1; - } - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* residual_in = ((residual_in_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* x_in = ((x_in_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* weight_t = ((weight_t_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* yp_out = ((yp_out_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* rp_out = ((rp_out_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* residual_out = ((residual_out_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - bool mhc_fused_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_fused_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_fused_tilelang_residual_in_is_null = (residual_in == NULL); - if (!(!mhc_fused_tilelang_residual_in_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang.residual_in is expected to have non-NULL pointer"); - return -1; - } - bool mhc_fused_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_fused_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_fused_tilelang_x_in_is_null = (x_in == NULL); - if (!(!mhc_fused_tilelang_x_in_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang.x_in is expected to have non-NULL pointer"); - return -1; - } - bool mhc_fused_tilelang_weight_t_is_null = (weight_t == NULL); - if (!(!mhc_fused_tilelang_weight_t_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang.weight_t is expected to have non-NULL pointer"); - return -1; - } - bool mhc_fused_tilelang_yp_out_is_null = (yp_out == NULL); - if (!(!mhc_fused_tilelang_yp_out_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang.yp_out is expected to have non-NULL pointer"); - return -1; - } - bool mhc_fused_tilelang_rp_out_is_null = (rp_out == NULL); - if (!(!mhc_fused_tilelang_rp_out_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang.rp_out is expected to have non-NULL pointer"); - return -1; - } - bool mhc_fused_tilelang_residual_out_is_null = (residual_out == NULL); - if (!(!mhc_fused_tilelang_residual_out_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_fused_tilelang.residual_out is expected to have non-NULL pointer"); - return -1; - } - void* mhc_fused_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_fused_tilelang_residual_in_shape = (((DLTensor*)residual_in)[0].shape); - void* mhc_fused_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_fused_tilelang_x_in_shape = (((DLTensor*)x_in)[0].shape); - void* mhc_fused_tilelang_weight_t_shape = (((DLTensor*)weight_t)[0].shape); - void* mhc_fused_tilelang_yp_out_shape = (((DLTensor*)yp_out)[0].shape); - void* mhc_fused_tilelang_rp_out_shape = (((DLTensor*)rp_out)[0].shape); - void* mhc_fused_tilelang_residual_out_shape = (((DLTensor*)residual_out)[0].shape); - if ((((DLTensor*)comb_mix)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_fused_tilelang.comb_mix, mhc_fused_tilelang.residual_in, mhc_fused_tilelang.post_mix, mhc_fused_tilelang.x_in, mhc_fused_tilelang.yp_out, mhc_fused_tilelang.rp_out, mhc_fused_tilelang.residual_out"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]); - void* mhc_fused_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - int32_t dev_id = (((DLTensor*)comb_mix)[0].device.device_id); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)residual_in)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual_in)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_fused_tilelang_residual_in_strides = (((DLTensor*)residual_in)[0].strides); - void* residual_in_1 = (((DLTensor*)residual_in)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_fused_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)x_in)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)x_in)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_fused_tilelang_x_in_strides = (((DLTensor*)x_in)[0].strides); - void* x_in_1 = (((DLTensor*)x_in)[0].data); - if ((((DLTensor*)weight_t)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)weight_t)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_fused_tilelang_weight_t_strides = (((DLTensor*)weight_t)[0].strides); - void* weight_t_1 = (((DLTensor*)weight_t)[0].data); - if ((((DLTensor*)yp_out)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)yp_out)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable split_k requires at least one non-null buffer among: mhc_fused_tilelang.yp_out, mhc_fused_tilelang.rp_out"); - return -1; - } - int32_t split_k = ((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_shape)[0]); - void* mhc_fused_tilelang_yp_out_strides = (((DLTensor*)yp_out)[0].strides); - void* yp_out_1 = (((DLTensor*)yp_out)[0].data); - if ((((DLTensor*)rp_out)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)rp_out)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_fused_tilelang_rp_out_strides = (((DLTensor*)rp_out)[0].strides); - void* rp_out_1 = (((DLTensor*)rp_out)[0].data); - if ((((DLTensor*)residual_out)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual_out)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_fused_tilelang_residual_out_strides = (((DLTensor*)residual_out)[0].strides); - void* residual_out_1 = (((DLTensor*)residual_out)[0].data); - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_9) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[2]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_fused_tilelang_comb_mix_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_fused_tilelang_comb_mix_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_fused_tilelang_comb_mix_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_strides)[1]); - } - if (condval_2 != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_3; - if ((mhc_fused_tilelang_comb_mix_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_3); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_fused_tilelang_comb_mix_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_strides)[0]); - } - if ((condval_4 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_15) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_17) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)residual_in)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual_in)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual_in)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual_in)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual_in)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual_in)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_18) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]) != ((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_fused_tilelang_residual_in_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_strides)[2]); - } - if (condval_5 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_6; - if ((mhc_fused_tilelang_residual_in_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_6); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_22) != 0) { - return -1; - } - } - int32_t condval_7; - if ((mhc_fused_tilelang_residual_in_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_strides)[1]); - } - if (condval_7 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_8; - if ((mhc_fused_tilelang_residual_in_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_8); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_23) != 0) { - return -1; - } - } - int32_t condval_9; - if ((mhc_fused_tilelang_residual_in_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_in_strides)[0]); - } - if ((condval_9 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_24) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual_in)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual_in)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)residual_in)[0].device.device_id) != (((DLTensor*)comb_mix)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual_in)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_26) != 0) { - return -1; - } - } - if ((((DLTensor*)residual_in)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual_in)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_27) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_in_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_28) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_29) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]) != ((int32_t)((int64_t*)mhc_fused_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_31) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_fused_tilelang_post_mix_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_fused_tilelang_post_mix_strides)[1]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_fused_tilelang_post_mix_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_fused_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_fused_tilelang_post_mix_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_fused_tilelang_post_mix_strides)[0]); - } - if ((condval_12 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_33) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_34) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)comb_mix)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_35) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_36) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_37) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)x_in)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)x_in)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)x_in)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)x_in)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)x_in)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)x_in)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_38) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]) != ((int32_t)((int64_t*)mhc_fused_tilelang_x_in_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_x_in_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_x_in_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_x_in_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_40) != 0) { - return -1; - } - } - int32_t condval_13; - if ((mhc_fused_tilelang_x_in_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_fused_tilelang_x_in_strides)[1]); - } - if (condval_13 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_14; - if ((mhc_fused_tilelang_x_in_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_fused_tilelang_x_in_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_14); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_41) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_fused_tilelang_x_in_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_fused_tilelang_x_in_strides)[0]); - } - if ((condval_15 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_42) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)x_in)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)x_in)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_43) != 0) { - return -1; - } - } - if ((((DLTensor*)x_in)[0].device.device_id) != (((DLTensor*)comb_mix)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)x_in)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if ((((DLTensor*)x_in)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)x_in)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_45) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (x_in_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"x_in"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_46) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)weight_t)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)weight_t)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)weight_t)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)weight_t)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)weight_t)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)weight_t)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_47) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_48) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_49) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - int32_t condval_16; - if ((mhc_fused_tilelang_weight_t_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_strides)[2]); - } - if (condval_16 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_17; - if ((mhc_fused_tilelang_weight_t_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_17); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_51) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_fused_tilelang_weight_t_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_strides)[1]); - } - if (condval_18 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_fused_tilelang_weight_t_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_52) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_fused_tilelang_weight_t_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_strides)[0]); - } - if (condval_20 != 16384) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16384; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_21; - if ((mhc_fused_tilelang_weight_t_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_fused_tilelang_weight_t_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_21); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_53) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)weight_t)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)weight_t)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_54) != 0) { - return -1; - } - } - if ((((DLTensor*)weight_t)[0].device.device_id) != (((DLTensor*)comb_mix)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)weight_t)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - if ((((DLTensor*)weight_t)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)weight_t)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_56) != 0) { - return -1; - } - } - if (weight_t_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"weight_t"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((((((DLTensor*)yp_out)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)yp_out)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)yp_out)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)yp_out)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)yp_out)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)yp_out)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_58) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]) != ((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_60) != 0) { - return -1; - } - } - int32_t condval_22; - if ((mhc_fused_tilelang_yp_out_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_strides)[2]); - } - if (condval_22 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_23; - if ((mhc_fused_tilelang_yp_out_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_23); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_61) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_fused_tilelang_yp_out_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_strides)[1]); - } - if ((condval_24 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_62) != 0) { - return -1; - } - } - int32_t condval_25; - if ((mhc_fused_tilelang_yp_out_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_strides)[0]); - } - if ((((num_tokens * 24) != condval_25) && (num_tokens != 0)) && (split_k != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_63) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)yp_out)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)yp_out)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_64) != 0) { - return -1; - } - } - if ((((DLTensor*)yp_out)[0].device.device_id) != (((DLTensor*)comb_mix)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)yp_out)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - if ((((DLTensor*)yp_out)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)yp_out)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_66) != 0) { - return -1; - } - } - if ((split_k != 0) && (num_tokens != 0)) { - if (yp_out_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"yp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_67) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)rp_out)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)rp_out)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)rp_out)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)rp_out)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)rp_out)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)rp_out)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_68) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_shape)[0]) != ((int32_t)((int64_t*)mhc_fused_tilelang_rp_out_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_rp_out_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_yp_out_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_69) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]) != ((int32_t)((int64_t*)mhc_fused_tilelang_rp_out_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_rp_out_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_70) != 0) { - return -1; - } - } - int32_t condval_26; - if ((mhc_fused_tilelang_rp_out_strides == NULL)) { - condval_26 = 1; - } else { - condval_26 = ((int32_t)((int64_t*)mhc_fused_tilelang_rp_out_strides)[1]); - } - if ((condval_26 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_71) != 0) { - return -1; - } - } - int32_t condval_27; - if ((mhc_fused_tilelang_rp_out_strides == NULL)) { - condval_27 = 1; - } else { - condval_27 = ((int32_t)((int64_t*)mhc_fused_tilelang_rp_out_strides)[0]); - } - if (((((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]) != condval_27) && (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]) != 0)) && (split_k != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_72) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)rp_out)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)rp_out)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_73) != 0) { - return -1; - } - } - if ((((DLTensor*)rp_out)[0].device.device_id) != (((DLTensor*)comb_mix)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)rp_out)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - if ((((DLTensor*)rp_out)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)rp_out)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_75) != 0) { - return -1; - } - } - if ((split_k != 0) && (num_tokens != 0)) { - if (rp_out_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"rp_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_76) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)residual_out)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual_out)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual_out)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual_out)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual_out)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual_out)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_77) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0]) != ((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_78) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_79) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_80) != 0) { - return -1; - } - } - int32_t condval_28; - if ((mhc_fused_tilelang_residual_out_strides == NULL)) { - condval_28 = 1; - } else { - condval_28 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_strides)[2]); - } - if (condval_28 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_29; - if ((mhc_fused_tilelang_residual_out_strides == NULL)) { - condval_29 = 1; - } else { - condval_29 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_29); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_30; - if ((mhc_fused_tilelang_residual_out_strides == NULL)) { - condval_30 = 1; - } else { - condval_30 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_strides)[1]); - } - if (condval_30 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_31; - if ((mhc_fused_tilelang_residual_out_strides == NULL)) { - condval_31 = 1; - } else { - condval_31 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_31); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - int32_t condval_32; - if ((mhc_fused_tilelang_residual_out_strides == NULL)) { - condval_32 = 1; - } else { - condval_32 = ((int32_t)((int64_t*)mhc_fused_tilelang_residual_out_strides)[0]); - } - if ((condval_32 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_83) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual_out)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual_out)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)residual_out)[0].device.device_id) != (((DLTensor*)comb_mix)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual_out)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_85) != 0) { - return -1; - } - } - if ((((DLTensor*)residual_out)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual_out)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_86) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_out_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_fused_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual_out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_87) != 0) { - return -1; - } - } - } else { - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_88) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = post_mix_1; - if (residual_in_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = residual_in_1; - if (residual_out_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = residual_out_1; - if (rp_out_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = rp_out_1; - if (weight_t_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = weight_t_1; - if (x_in_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = x_in_1; - if (yp_out_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = yp_out_1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)split_k); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)12); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)split_k); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)256); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[16].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[16].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[16].v_int64) = ((int64_t)176); - (((TVMFFIAny*)stack_ffi_any)[17].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[17].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[17].v_int64) = (int64_t)0; - if (mhc_fused_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_fused_tilelang_kernel", &mhc_fused_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_89; - result_89.type_index = kTVMFFINone; - result_89.zero_padding = 0; - result_89.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_fused_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 17, &result_89) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_fused_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/96e9b9b5cf258ab70fb96c06c4ba12fa70819bfb1b8c147e903abf420053524b/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/96e9b9b5cf258ab70fb96c06c4ba12fa70819bfb1b8c147e903abf420053524b/params.pkl deleted file mode 100644 index 2a45bcde1473ac97d9936a6f472c0d5b315031a0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2862 zcmeHIL2DC16pn2Y1F<1cN~<8~cxfRlO$#1!_O@7xiWe!vZ1yEH&F;)HJE<*U!5->q zZW;fb;9v3WCaKM$2-bs`>}`hcz4yIu-uw1b?aPa1rTEU=zTGh*f{4kX2XZJRcnQ-i zcDCt*mJpp5r>Q%*b{V2;p}9)H*_OG8>0lr{I=kTYsk09Ukz(ff$lbVWcakBB&S7(u zeY6PcD~(wm)4m2~fcA?lbf8sC%{Ym`O|e)B-FU?edzY7=I)B4|=07m*jY~hfz}8aE z@E-f&E!(6LK2jB&L*F2@NaGoyJ-AG?vzzak#_Jo5MT&LKWQg_bw+Yco87gE5m!x;x zAyS2qlHP|2$$KU}a-t-VtaE$L`+sY+kVDcTA(BIk9AgJ+1Twvg#Nww~#5ws}=!B#R zlcaZuvGW2#qIx8|QzEC0e19K=bXklfyPbk9Wl}1`j8ItND9g?=mj-^-ufjOso3qE{ zb>6Uzpm?>-G_sV -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void hc_head_fuse_tilelang_kernel(const float* fn, const float* hc_base, const float* hc_scale, bfloat16_t* out, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(128, 1) hc_head_fuse_tilelang_kernel(const float* fn, const float* hc_base, const float* hc_scale, bfloat16_t* out, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float sqrsum_r[1]; - float mixes_r[4]; - bfloat16_t residual_local_cast[8]; - float x_local[8]; - float fn_local[8]; - float rsqrt_val[1]; - float xl[32]; - float ol[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t out_local_cast_2[8]; - bfloat16_t xs_local_cast_3[8]; - bfloat16_t out_local_cast_4[8]; - bfloat16_t xs_local_cast_5[8]; - bfloat16_t out_local_cast_6[8]; - cudaGridDependencySynchronize(); - sqrsum_r[0] = 0x0p+0f/*0.000000e+00*/; - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(mixes_r + 0) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - for (int m_c = 0; m_c < 4; ++m_c) { - for (int i_h = 0; i_h < 4; ++i_h) { - *(uint4*)(residual_local_cast + 0) = *(uint4*)(residual + ((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + (((int64_t)m_c) * (int64_t)4096)) + (((int64_t)i_h) * (int64_t)1024)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8))); - for (int i = 0; i < 2; ++i) { - float4 __1; - uint2 v_ = *(uint2*)(residual_local_cast + (i * 4)); - ((float2*)(&__1))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__1))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(x_local + (i * 4)) = __1; - } - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - sqrsum_r[0] = (sqrsum_r[0] + (x_local[i_1] * x_local[i_1])); - } - #pragma unroll - for (int m_m = 0; m_m < 4; ++m_m) { - #pragma unroll - for (int i_2 = 0; i_2 < 2; ++i_2) { - *(float4*)(fn_local + (i_2 * 4)) = *(float4*)(fn + (((((m_m * 16384) + (m_c * 4096)) + (i_h * 1024)) + (((int)threadIdx.x) * 8)) + (i_2 * 4))); - } - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - mixes_r[m_m] = (mixes_r[m_m] + (x_local[i_3] * fn_local[i_3])); - } - } - } - } - sqrsum_r[0] = tl::AllReduce>::run(sqrsum_r[0], (&(((float*)buf_dyn_shmem)[0]))); - __syncthreads(); - for (int __finred_0 = 0; __finred_0 < 4; ++__finred_0) { - mixes_r[__finred_0] = tl::AllReduce>::run(mixes_r[__finred_0], (&(((float*)buf_dyn_shmem)[0]))); - } - rsqrt_val[0] = rsqrtf(((sqrsum_r[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - __syncthreads(); - if ((((int)threadIdx.x) >> 2) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) & 3)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - (((mixes_r[(((int)threadIdx.x) & 3)] * rsqrt_val[0]) * hc_scale[0]) + hc_base[(((int)threadIdx.x) & 3)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - __syncthreads(); - #pragma unroll - for (int i_4 = 0; i_4 < 4; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_4 * 1024) + (((int)threadIdx.x) * 8)) + 256)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + (((int64_t)i_4) * (int64_t)4096)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_5 * 1024) + (((int)threadIdx.x) * 8)) + 4352)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + (((int64_t)i_5) * (int64_t)4096)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - __syncthreads(); - #pragma unroll - for (int i_6 = 0; i_6 < 4; ++i_6) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_6 * 1024)) + (((int)threadIdx.x) * 8)) + 256)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v__1 = *(uint2*)(xs_local_cast_1 + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__1))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__1))[1]); - *(float4*)(xl + ((i_6 * 8) + (vec * 4))) = __2; - } - } - __syncthreads(); - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + (i_7 * 1024)) + (((int)threadIdx.x) * 8)) + 256)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + (((int64_t)i_7) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_8 = 0; i_8 < 2; ++i_8) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_8 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - __syncthreads(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[i_hc]; - #pragma unroll - for (int i_9 = 0; i_9 < 8; ++i_9) { - ol[i_9] = (ol[i_9] + (pre * xl[((i_hc * 8) + i_9)])); - } - } - for (int i_10 = 0; i_10 < 2; ++i_10) { - uint2 __3; - float4 v__2 = *(float4*)(ol + (i_10 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__3))[0] = __float22bfloat162_rn(((float2*)(&v__2))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__3))[1] = __float22bfloat162_rn(((float2*)(&v__2))[1]); - *(uint2*)(out_local_cast_2 + (i_10 * 4)) = __3; - } - *(uint4*)(out + (((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h) * (int64_t)1024)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8))) = *(uint4*)(out_local_cast_2 + 0); - } - tl::cp_async_wait<1>(); - __syncthreads(); - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - *(uint4*)(xs_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_11 * 1024) + (((int)threadIdx.x) * 8)) + 256)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __4; - uint2 v__3 = *(uint2*)(xs_local_cast_3 + (vec_1 * 4)); - ((float2*)(&__4))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__3))[0]); - ((float2*)(&__4))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__3))[1]); - *(float4*)(xl + ((i_11 * 8) + (vec_1 * 4))) = __4; - } - } - #pragma unroll - for (int i_12 = 0; i_12 < 2; ++i_12) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_12 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[i_hc_1]; - #pragma unroll - for (int i_13 = 0; i_13 < 8; ++i_13) { - ol[i_13] = (ol[i_13] + (pre_1 * xl[((i_hc_1 * 8) + i_13)])); - } - } - for (int i_14 = 0; i_14 < 2; ++i_14) { - uint2 __5; - float4 v__4 = *(float4*)(ol + (i_14 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__4))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__4))[1]); - *(uint2*)(out_local_cast_4 + (i_14 * 4)) = __5; - } - *(uint4*)(out + (((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2048)) = *(uint4*)(out_local_cast_4 + 0); - tl::cp_async_wait<0>(); - __syncthreads(); - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - *(uint4*)(xs_local_cast_5 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_15 * 1024) + (((int)threadIdx.x) * 8)) + 4352)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __6; - uint2 v__5 = *(uint2*)(xs_local_cast_5 + (vec_2 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__5))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__5))[1]); - *(float4*)(xl + ((i_15 * 8) + (vec_2 * 4))) = __6; - } - } - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_16 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[i_hc_2]; - #pragma unroll - for (int i_17 = 0; i_17 < 8; ++i_17) { - ol[i_17] = (ol[i_17] + (pre_2 * xl[((i_hc_2 * 8) + i_17)])); - } - } - for (int i_18 = 0; i_18 < 2; ++i_18) { - uint2 __7; - float4 v__6 = *(float4*)(ol + (i_18 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__7))[0] = __float22bfloat162_rn(((float2*)(&v__6))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__7))[1] = __float22bfloat162_rn(((float2*)(&v__6))[1]); - *(uint2*)(out_local_cast_6 + (i_18 * 4)) = __7; - } - *(uint4*)(out + (((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)3072)) = *(uint4*)(out_local_cast_6 + 0); - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/bc130d0880c8dc66c0a86cfbe52a5accea0d7723ef84286eda1809080514e34c/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/bc130d0880c8dc66c0a86cfbe52a5accea0d7723ef84286eda1809080514e34c/host_kernel.cu deleted file mode 100644 index 4ceadb9..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/bc130d0880c8dc66c0a86cfbe52a5accea0d7723ef84286eda1809080514e34c/host_kernel.cu +++ /dev/null @@ -1,1828 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* hc_head_fuse_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_hc_head_fuse_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_hc_head_fuse_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[12]; - void* stack_ffi_any = stack; - if (!((num_args == 5))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "hc_head_fuse_tilelang: num_args should be 5", (long long)(num_args), (long long)(5)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "hc_head_fuse_tilelang: args pointer is NULL"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel hc_head_fuse_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t fn_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((fn_type_index == 0) || (fn_type_index == 4)) || (fn_type_index == 7)) || (64 <= fn_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel hc_head_fuse_tilelang input fn expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel hc_head_fuse_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel hc_head_fuse_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t out_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((out_type_index == 0) || (out_type_index == 4)) || (out_type_index == 7)) || (64 <= out_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel hc_head_fuse_tilelang input out expected pointer or tensor handle"); - return -1; - } - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* fn = ((fn_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* out = ((out_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - bool hc_head_fuse_tilelang_residual_is_null = (residual == NULL); - if (!(!hc_head_fuse_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "hc_head_fuse_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool hc_head_fuse_tilelang_fn_is_null = (fn == NULL); - if (!(!hc_head_fuse_tilelang_fn_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "hc_head_fuse_tilelang.fn is expected to have non-NULL pointer"); - return -1; - } - bool hc_head_fuse_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!hc_head_fuse_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "hc_head_fuse_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool hc_head_fuse_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!hc_head_fuse_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "hc_head_fuse_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool hc_head_fuse_tilelang_out_is_null = (out == NULL); - if (!(!hc_head_fuse_tilelang_out_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "hc_head_fuse_tilelang.out is expected to have non-NULL pointer"); - return -1; - } - void* hc_head_fuse_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* hc_head_fuse_tilelang_fn_shape = (((DLTensor*)fn)[0].shape); - void* hc_head_fuse_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* hc_head_fuse_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* hc_head_fuse_tilelang_out_shape = (((DLTensor*)out)[0].shape); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: hc_head_fuse_tilelang.residual, hc_head_fuse_tilelang.out"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)hc_head_fuse_tilelang_residual_shape)[0]); - void* hc_head_fuse_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - int32_t dev_id = (((DLTensor*)residual)[0].device.device_id); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)fn)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"fn"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)fn)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* hc_head_fuse_tilelang_fn_strides = (((DLTensor*)fn)[0].strides); - void* fn_1 = (((DLTensor*)fn)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* hc_head_fuse_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* hc_head_fuse_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)out)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)out)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* hc_head_fuse_tilelang_out_strides = (((DLTensor*)out)[0].strides); - void* out_1 = (((DLTensor*)out)[0].data); - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_6) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)hc_head_fuse_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)hc_head_fuse_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_7) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)hc_head_fuse_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)hc_head_fuse_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_8) != 0) { - return -1; - } - } - int32_t condval; - if ((hc_head_fuse_tilelang_residual_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)hc_head_fuse_tilelang_residual_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((hc_head_fuse_tilelang_residual_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)hc_head_fuse_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_9) != 0) { - return -1; - } - } - int32_t condval_2; - if ((hc_head_fuse_tilelang_residual_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)hc_head_fuse_tilelang_residual_strides)[1]); - } - if (condval_2 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_3; - if ((hc_head_fuse_tilelang_residual_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)hc_head_fuse_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_3); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_10) != 0) { - return -1; - } - } - int32_t condval_4; - if ((hc_head_fuse_tilelang_residual_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)hc_head_fuse_tilelang_residual_strides)[0]); - } - if ((condval_4 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_11) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_12) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_13) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)fn)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)fn)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)fn)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"fn"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)fn)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)fn)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)fn)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_15) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)hc_head_fuse_tilelang_fn_shape)[0]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"fn"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)hc_head_fuse_tilelang_fn_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_16) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)hc_head_fuse_tilelang_fn_shape)[1]) != 16384) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"fn"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16384; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)hc_head_fuse_tilelang_fn_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_17) != 0) { - return -1; - } - } - int32_t condval_5; - if ((hc_head_fuse_tilelang_fn_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)hc_head_fuse_tilelang_fn_strides)[1]); - } - if (condval_5 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"fn"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_6; - if ((hc_head_fuse_tilelang_fn_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)hc_head_fuse_tilelang_fn_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_6); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_18) != 0) { - return -1; - } - } - int32_t condval_7; - if ((hc_head_fuse_tilelang_fn_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)hc_head_fuse_tilelang_fn_strides)[0]); - } - if (condval_7 != 16384) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"fn"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16384; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_8; - if ((hc_head_fuse_tilelang_fn_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)hc_head_fuse_tilelang_fn_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_8); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_19) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)fn)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"fn"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)fn)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_20) != 0) { - return -1; - } - } - if ((((DLTensor*)fn)[0].device.device_id) != (((DLTensor*)residual)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"fn"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)fn)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - if ((((DLTensor*)fn)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"fn"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)fn)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_22) != 0) { - return -1; - } - } - if (fn_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"fn"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_23) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_24) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)hc_head_fuse_tilelang_hc_scale_shape)[0]) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)hc_head_fuse_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_25) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_26) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)residual)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_27) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_28) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_29) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_30) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)hc_head_fuse_tilelang_hc_base_shape)[0]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)hc_head_fuse_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_31) != 0) { - return -1; - } - } - int32_t condval_9; - if ((hc_head_fuse_tilelang_hc_base_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)hc_head_fuse_tilelang_hc_base_strides)[0]); - } - if (condval_9 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_10; - if ((hc_head_fuse_tilelang_hc_base_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)hc_head_fuse_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_10); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)residual)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_34) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_35) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_36) != 0) { - return -1; - } - } - if ((((((DLTensor*)out)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)out)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)out)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)out)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)out)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)out)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_37) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)hc_head_fuse_tilelang_residual_shape)[0]) != ((int32_t)((int64_t*)hc_head_fuse_tilelang_out_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)hc_head_fuse_tilelang_out_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)hc_head_fuse_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_38) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)hc_head_fuse_tilelang_out_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)hc_head_fuse_tilelang_out_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - int32_t condval_11; - if ((hc_head_fuse_tilelang_out_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)hc_head_fuse_tilelang_out_strides)[1]); - } - if (condval_11 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_12; - if ((hc_head_fuse_tilelang_out_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)hc_head_fuse_tilelang_out_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_12); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_40) != 0) { - return -1; - } - } - int32_t condval_13; - if ((hc_head_fuse_tilelang_out_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)hc_head_fuse_tilelang_out_strides)[0]); - } - if ((condval_13 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_41) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)out)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)out)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_42) != 0) { - return -1; - } - } - if ((((DLTensor*)out)[0].device.device_id) != (((DLTensor*)residual)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)out)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - if ((((DLTensor*)out)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)out)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_44) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (out_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"hc_head_fuse_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"out"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_45) != 0) { - return -1; - } - } - } else { - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_46) != 0) { - return -1; - } - if (fn_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = fn_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = hc_scale_1; - if (out_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = out_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = ((int64_t)128); - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)16896); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = (int64_t)0; - if (hc_head_fuse_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "hc_head_fuse_tilelang_kernel", &hc_head_fuse_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(hc_head_fuse_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 11, &result_47) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_hc_head_fuse_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/bc130d0880c8dc66c0a86cfbe52a5accea0d7723ef84286eda1809080514e34c/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/bc130d0880c8dc66c0a86cfbe52a5accea0d7723ef84286eda1809080514e34c/params.pkl deleted file mode 100644 index e2fd8adbac4064ed18e3b107edff0df6c84504c2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1000 zcmd^7O-my|5LNSGzz+hlh=QPfQV7GEfQOts?5$Z5UG^f<^kgdOc4xYWp3a&j46Fw| z^)2ns6aR`elSEzjAb7QJRlHa4RlWXOyML%Gl<(5-yDwJ85OXyMKnIv!;$0)`$C}&2BY*2}{7H^5Hp9(eZgtM< zX@g_7SkC~rzbb`;F*q)Se;0+P3$>*4_C)<`uVEfS$Q#Hb-|pnrkNUL#Tb)X1bT{|p_IEai%Hzan=g zAeHU3N&ZTru#JA-C!t-Qpvl8l>6US&wB=T6RA}eKo^y3fH=_3EAGOfO){`b}9uK)QKrE9S*cBWa2z0VNx IG-3#z0OQeMqyPW_ diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/c0ef72219d5280025634d12a15beae51ea48dcaded4066ba7f411defa4b60bff/device_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/c0ef72219d5280025634d12a15beae51ea48dcaded4066ba7f411defa4b60bff/device_kernel.cu deleted file mode 100644 index 5bfbedc..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/c0ef72219d5280025634d12a15beae51ea48dcaded4066ba7f411defa4b60bff/device_kernel.cu +++ /dev/null @@ -1,424 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float mixes[1]; - float rms[1]; - float cm[1]; - float row_max[1]; - float row_sum[1]; - float col_sum[1]; - float sumsq_per_pos[16]; - float sumsq[1]; - float rsqrt_norm[1]; - float xl[64]; - float ol[16]; - bfloat16_t xs_local_cast[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t xs_local_cast_2[8]; - float w_local[16]; - float ol_1[16]; - bfloat16_t w_shared_local_cast_3[8]; - bfloat16_t output_shared_local_cast_4[8]; - bfloat16_t layer_input_local_cast_5[8]; - bfloat16_t w_shared_local_cast_6[8]; - bfloat16_t output_shared_local_cast_7[8]; - bfloat16_t layer_input_local_cast_8[8]; - bfloat16_t w_shared_local_cast_9[8]; - bfloat16_t output_shared_local_cast_10[8]; - bfloat16_t layer_input_local_cast_11[8]; - mixes[0] = 0x0p+0f/*0.000000e+00*/; - rms[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - for (int i_split = 0; i_split < 9; ++i_split) { - rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]); - } - rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - mixes[0] = 0x0p+0f/*0.000000e+00*/; - for (int i_split_1 = 0; i_split_1 < 9; ++i_split_1) { - mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]); - } - mixes[0] = (mixes[0] * rms[0]); - if ((((int)threadIdx.x) / 24) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0]; - } - __syncthreads(); - if (((int)threadIdx.x) < 32) { - if (((int)threadIdx.x) < 4) { - post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/); - } - cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]); - row_max[0] = -CUDART_INF_F; - row_max[0] = max(row_max[0], cm[0]); - row_max[0] = tl::AllReduce>::run(row_max[0]); - cm[0] = expf((cm[0] - row_max[0])); - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - for (int __1 = 0; __1 < 19; ++__1) { - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - } - if ((((int)threadIdx.x) >> 4) == 0) { - comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0]; - } - } else { - if (((int)threadIdx.x) < 36) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - #pragma unroll - for (int i = 0; i < 4; ++i) { - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_2 = 0; i_2 < 8; ++i_2) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_4 = 0; i_4 < 8; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - tl::__sync_thread_partial<3, 64>(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)]; - #pragma unroll - for (int i_6 = 0; i_6 < 16; ++i_6) { - ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)])); - } - } - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - float4 __3; - float4 v__1 = *(float4*)(sumsq_per_pos + (i_7 * 4)); - float4 __4; - float4 v__2 = *(float4*)(ol + (i_7 * 4)); - __4.x = (v__2.x*v__2.x); - __4.y = (v__2.y*v__2.y); - __4.z = (v__2.z*v__2.z); - __4.w = (v__2.w*v__2.w); - __3.x = (v__1.x+__4.x); - __3.y = (v__1.y+__4.y); - __3.z = (v__1.z+__4.z); - __3.w = (v__1.w+__4.w); - *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3; - uint2 __5; - float4 v__3 = *(float4*)(ol + (i_7 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __5; - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_8 = 0; i_8 < 8; ++i_8) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __6; - uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]); - *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __6; - } - } - #pragma unroll - for (int i_9 = 0; i_9 < 4; ++i_9) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)]; - #pragma unroll - for (int i_10 = 0; i_10 < 16; ++i_10) { - ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)])); - } - } - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - float4 __7; - float4 v__5 = *(float4*)(sumsq_per_pos + (i_11 * 4)); - float4 __8; - float4 v__6 = *(float4*)(ol + (i_11 * 4)); - __8.x = (v__6.x*v__6.x); - __8.y = (v__6.y*v__6.y); - __8.z = (v__6.z*v__6.z); - __8.w = (v__6.w*v__6.w); - __7.x = (v__5.x+__8.x); - __7.y = (v__5.y+__8.y); - __7.z = (v__5.z+__8.z); - __7.w = (v__5.w+__8.w); - *(float4*)(sumsq_per_pos + (i_11 * 4)) = __7; - uint2 __9; - float4 v__7 = *(float4*)(ol + (i_11 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__9))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__9))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __9; - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_12 = 0; i_12 < 8; ++i_12) { - *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __10; - uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4)); - ((float2*)(&__10))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]); - ((float2*)(&__10))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]); - *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __10; - } - } - #pragma unroll - for (int i_13 = 0; i_13 < 4; ++i_13) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)]; - #pragma unroll - for (int i_14 = 0; i_14 < 16; ++i_14) { - ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)])); - } - } - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - float4 __11; - float4 v__9 = *(float4*)(sumsq_per_pos + (i_15 * 4)); - float4 __12; - float4 v__10 = *(float4*)(ol + (i_15 * 4)); - __12.x = (v__10.x*v__10.x); - __12.y = (v__10.y*v__10.y); - __12.z = (v__10.z*v__10.z); - __12.w = (v__10.w*v__10.w); - __11.x = (v__9.x+__12.x); - __11.y = (v__9.y+__12.y); - __11.z = (v__9.z+__12.z); - __11.w = (v__9.w+__12.w); - *(float4*)(sumsq_per_pos + (i_15 * 4)) = __11; - uint2 __13; - float4 v__11 = *(float4*)(ol + (i_15 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__13))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__13))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __13; - } - sumsq[0] = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int rv = 0; rv < 16; ++rv) { - sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]); - } - tl::__sync_thread_partial<3, 64>(); - sumsq[0] = tl::AllReduce>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192]))); - rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_17 = 0; i_17 < 2; ++i_17) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)]))); - } - tl::cp_async_commit(); - for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_18 = 0; i_18 < 2; ++i_18) { - *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_3 = 0; vec_3 < 2; ++vec_3) { - float4 __14; - uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4)); - ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]); - ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]); - *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __14; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_19 = 0; i_19 < 2; ++i_19) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)]))); - } - tl::cp_async_commit(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_20 = 0; i_20 < 2; ++i_20) { - *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984)); - for (int vec_4 = 0; vec_4 < 2; ++vec_4) { - float4 __15; - float4 __16; - float4 __17; - uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4)); - ((float2*)(&__17))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]); - ((float2*)(&__17))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]); - float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __16.x = (__17.x*v__14.x); - __16.y = (__17.y*v__14.y); - __16.z = (__17.z*v__14.z); - __16.w = (__17.w*v__14.w); - float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4))); - __15.x = (__16.x*v__15.x); - __15.y = (__16.y*v__15.y); - __15.z = (__16.z*v__15.z); - __15.w = (__16.w*v__15.w); - *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __15; - } - } - #pragma unroll - for (int i_21 = 0; i_21 < 2; ++i_21) { - for (int vec_5 = 0; vec_5 < 2; ++vec_5) { - uint2 __18; - float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__18))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__18))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]); - *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __18; - } - *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0); - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_22 = 0; i_22 < 2; ++i_22) { - *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_6 = 0; vec_6 < 2; ++vec_6) { - float4 __19; - uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4)); - ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]); - ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]); - *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __19; - } - } - #pragma unroll - for (int i_23 = 0; i_23 < 2; ++i_23) { - *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032)); - for (int vec_7 = 0; vec_7 < 2; ++vec_7) { - float4 __20; - float4 __21; - float4 __22; - uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4)); - ((float2*)(&__22))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]); - ((float2*)(&__22))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]); - float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __21.x = (__22.x*v__19.x); - __21.y = (__22.y*v__19.y); - __21.z = (__22.z*v__19.z); - __21.w = (__22.w*v__19.w); - float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4))); - __20.x = (__21.x*v__20.x); - __20.y = (__21.y*v__20.y); - __20.z = (__21.z*v__20.z); - __20.w = (__21.w*v__20.w); - *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __20; - } - } - #pragma unroll - for (int i_24 = 0; i_24 < 2; ++i_24) { - for (int vec_8 = 0; vec_8 < 2; ++vec_8) { - uint2 __23; - float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__23))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__23))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]); - *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __23; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0); - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_25 = 0; i_25 < 2; ++i_25) { - *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104)); - for (int vec_9 = 0; vec_9 < 2; ++vec_9) { - float4 __24; - uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4)); - ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]); - ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]); - *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __24; - } - } - #pragma unroll - for (int i_26 = 0; i_26 < 2; ++i_26) { - *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056)); - for (int vec_10 = 0; vec_10 < 2; ++vec_10) { - float4 __25; - float4 __26; - float4 __27; - uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4)); - ((float2*)(&__27))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]); - ((float2*)(&__27))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]); - float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __26.x = (__27.x*v__24.x); - __26.y = (__27.y*v__24.y); - __26.z = (__27.z*v__24.z); - __26.w = (__27.w*v__24.w); - float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4))); - __25.x = (__26.x*v__25.x); - __25.y = (__26.y*v__25.y); - __25.z = (__26.z*v__25.z); - __25.w = (__26.w*v__25.w); - *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __25; - } - } - #pragma unroll - for (int i_27 = 0; i_27 < 2; ++i_27) { - for (int vec_11 = 0; vec_11 < 2; ++vec_11) { - uint2 __28; - float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__28))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__28))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]); - *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __28; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/c0ef72219d5280025634d12a15beae51ea48dcaded4066ba7f411defa4b60bff/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/c0ef72219d5280025634d12a15beae51ea48dcaded4066ba7f411defa4b60bff/host_kernel.cu deleted file mode 100644 index 2f76fb5..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/c0ef72219d5280025634d12a15beae51ea48dcaded4066ba7f411defa4b60bff/host_kernel.cu +++ /dev/null @@ -1,3359 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_pre_big_fuse_with_norm_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[16]; - void* stack_ffi_any = stack; - if (!((num_args == 9))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_pre_big_fuse_with_norm_tilelang: num_args should be 9", (long long)(num_args), (long long)(9)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang: args pointer is NULL"); - return -1; - } - int32_t gemm_out_mul_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((gemm_out_mul_type_index == 0) || (gemm_out_mul_type_index == 4)) || (gemm_out_mul_type_index == 7)) || (64 <= gemm_out_mul_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_mul expected pointer or tensor handle"); - return -1; - } - int32_t gemm_out_sqrsum_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((gemm_out_sqrsum_type_index == 0) || (gemm_out_sqrsum_type_index == 4)) || (gemm_out_sqrsum_type_index == 7)) || (64 <= gemm_out_sqrsum_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_sqrsum expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t layer_input_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((layer_input_type_index == 0) || (layer_input_type_index == 4)) || (layer_input_type_index == 7)) || (64 <= layer_input_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input layer_input expected pointer or tensor handle"); - return -1; - } - int32_t norm_weight_type_index = (((TVMFFIAny*)args)[8].type_index); - if (!(((((norm_weight_type_index == 0) || (norm_weight_type_index == 4)) || (norm_weight_type_index == 7)) || (64 <= norm_weight_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input norm_weight expected pointer or tensor handle"); - return -1; - } - void* gemm_out_mul = ((gemm_out_mul_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* gemm_out_sqrsum = ((gemm_out_sqrsum_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* layer_input = ((layer_input_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - void* norm_weight = ((norm_weight_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[8].v_ptr) + 24)) : (((TVMFFIAny*)args)[8].v_ptr)); - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null = (gemm_out_mul == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null = (gemm_out_sqrsum == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_residual_is_null = (residual == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null = (layer_input == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.layer_input is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null = (norm_weight == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.norm_weight is expected to have non-NULL pointer"); - return -1; - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape = (((DLTensor*)gemm_out_mul)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape = (((DLTensor*)gemm_out_sqrsum)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape = (((DLTensor*)layer_input)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape = (((DLTensor*)norm_weight)[0].shape); - if ((((DLTensor*)gemm_out_mul)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul, mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum, mhc_pre_big_fuse_with_norm_tilelang.residual, mhc_pre_big_fuse_with_norm_tilelang.post_mix, mhc_pre_big_fuse_with_norm_tilelang.comb_mix, mhc_pre_big_fuse_with_norm_tilelang.layer_input"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides = (((DLTensor*)gemm_out_mul)[0].strides); - int32_t dev_id = (((DLTensor*)gemm_out_mul)[0].device.device_id); - void* gemm_out_mul_1 = (((DLTensor*)gemm_out_mul)[0].data); - if ((((DLTensor*)gemm_out_sqrsum)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides = (((DLTensor*)gemm_out_sqrsum)[0].strides); - void* gemm_out_sqrsum_1 = (((DLTensor*)gemm_out_sqrsum)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)comb_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)layer_input)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides = (((DLTensor*)layer_input)[0].strides); - void* layer_input_1 = (((DLTensor*)layer_input)[0].data); - if ((((DLTensor*)norm_weight)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_9) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides = (((DLTensor*)norm_weight)[0].strides); - void* norm_weight_1 = (((DLTensor*)norm_weight)[0].data); - if ((((((DLTensor*)gemm_out_mul)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_mul)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_mul)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0]) != 9) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)9; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[1]); - } - if ((condval_2 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - int32_t condval_3; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[0]); - } - if (((num_tokens * 24) != condval_3) && (num_tokens != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_15) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_mul)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_mul)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_17) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_18) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)gemm_out_sqrsum)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0]) != 9) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)9; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[1]); - } - if ((condval_4 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_22) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[0]); - } - if ((((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != condval_5) && (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_23) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_sqrsum)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_26) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_27) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_28) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_29) != 0) { - return -1; - } - } - int32_t condval_6; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - if (condval_6 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_7; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_7); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_34) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_35) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_36) != 0) { - return -1; - } - } - int32_t condval_8; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - if (condval_8 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_9; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_9); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_38) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_40) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_41) != 0) { - return -1; - } - } - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_42) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_46) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - if (condval_12 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_13; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_13); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_47) != 0) { - return -1; - } - } - int32_t condval_14; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[0]); - } - if ((condval_14 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_48) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_49) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_51) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_52) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_53) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_54) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - if (condval_15 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_16; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_16); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_56) != 0) { - return -1; - } - } - int32_t condval_17; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[0]); - } - if ((condval_17 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_58) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_60) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_61) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_62) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_63) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1]) != 16) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_64) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - if (condval_18 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[0]); - } - if ((condval_20 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_66) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_67) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_68) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_69) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_70) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)layer_input)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)layer_input)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)layer_input)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_71) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_72) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_73) != 0) { - return -1; - } - } - int32_t condval_21; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - if (condval_21 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_22; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_22); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - int32_t condval_23; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[0]); - } - if ((condval_23 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_75) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)layer_input)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_76) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_77) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_78) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_79) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)norm_weight)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)norm_weight)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)norm_weight)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_80) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - if (condval_24 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_25; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_25); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)norm_weight)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_83) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_85) != 0) { - return -1; - } - } - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_86) != 0) { - return -1; - } - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_87) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = gemm_out_mul_1; - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = gemm_out_sqrsum_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = hc_scale_1; - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = layer_input_1; - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = norm_weight_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = post_mix_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)96); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)29040); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = (int64_t)0; - if (mhc_pre_big_fuse_with_norm_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_pre_big_fuse_with_norm_tilelang_kernel", &mhc_pre_big_fuse_with_norm_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_pre_big_fuse_with_norm_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 15, &result_88) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/c0ef72219d5280025634d12a15beae51ea48dcaded4066ba7f411defa4b60bff/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/c0ef72219d5280025634d12a15beae51ea48dcaded4066ba7f411defa4b60bff/params.pkl deleted file mode 100644 index c9f15f6efb8b07a166582ff649dcea0cd8ff1fc3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2200 zcmeHIO=}x55KZgYb#M|4r8Er`8l4<4OXENeKKCXPLR)$%MrgfbOYN?-XyqowEHsDY zq@Jw)zW$ZYu0NW_fd*fly=ize@6DSX=|}a~&-zCB^?b{{urh|2t6>0YC=~=LH$3sK zLBS}9&&$)qA71$!(bdR)PrLEfehF*AZiouo(*En?QzFn`zwEzBZ|#nGnCt& zr}eeLDz8}I0Jp&UWfe9sI$?H{#^5JNb_ZR5$1VR@&`&&lO5e~Y*1z}JjSpQl6CC5% z58V>GuE7*sFdcq@zD00ZD6rB5aG4tKDxbK)@HJ+m$eQ3PLPqy{OpMl+Nfp5*=^nR< z(h+2&`*}=?u1S}iX$2&2FV2MnAFPpTNZKSqd5BRX>_LA8mEA@n`MHrvA%6iKlPu+m zbl+m`j6f>%kL0%~3fZXl|4As9Cn)l$U8-eVDQ&rx8W}oy-phR!(mfc3qB0$(sEPX0 z%*2jpw#4&){S_~#&hCrXQ`5x3I=Hp6SG2-cwVCeSO|6IJS{FW5?0WR7xLzN0Q!f2ik9%0WQTIM8_iW+A-NIzN@*wyFv8nbG diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/e8bfe015e13b0beed5fbf1df07913214e93cd08d927e1e810f022bb8d28739b0/device_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/e8bfe015e13b0beed5fbf1df07913214e93cd08d927e1e810f022bb8d28739b0/device_kernel.cu deleted file mode 100644 index 83c3db9..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/e8bfe015e13b0beed5fbf1df07913214e93cd08d927e1e810f022bb8d28739b0/device_kernel.cu +++ /dev/null @@ -1,424 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float mixes[1]; - float rms[1]; - float cm[1]; - float row_max[1]; - float row_sum[1]; - float col_sum[1]; - float sumsq_per_pos[16]; - float sumsq[1]; - float rsqrt_norm[1]; - float xl[64]; - float ol[16]; - bfloat16_t xs_local_cast[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t xs_local_cast_2[8]; - float w_local[16]; - float ol_1[16]; - bfloat16_t w_shared_local_cast_3[8]; - bfloat16_t output_shared_local_cast_4[8]; - bfloat16_t layer_input_local_cast_5[8]; - bfloat16_t w_shared_local_cast_6[8]; - bfloat16_t output_shared_local_cast_7[8]; - bfloat16_t layer_input_local_cast_8[8]; - bfloat16_t w_shared_local_cast_9[8]; - bfloat16_t output_shared_local_cast_10[8]; - bfloat16_t layer_input_local_cast_11[8]; - mixes[0] = 0x0p+0f/*0.000000e+00*/; - rms[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - for (int i_split = 0; i_split < 3; ++i_split) { - rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]); - } - rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - mixes[0] = 0x0p+0f/*0.000000e+00*/; - for (int i_split_1 = 0; i_split_1 < 3; ++i_split_1) { - mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]); - } - mixes[0] = (mixes[0] * rms[0]); - if ((((int)threadIdx.x) / 24) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0]; - } - __syncthreads(); - if (((int)threadIdx.x) < 32) { - if (((int)threadIdx.x) < 4) { - post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/); - } - cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]); - row_max[0] = -CUDART_INF_F; - row_max[0] = max(row_max[0], cm[0]); - row_max[0] = tl::AllReduce>::run(row_max[0]); - cm[0] = expf((cm[0] - row_max[0])); - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - for (int __1 = 0; __1 < 19; ++__1) { - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - } - if ((((int)threadIdx.x) >> 4) == 0) { - comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0]; - } - } else { - if (((int)threadIdx.x) < 36) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - #pragma unroll - for (int i = 0; i < 4; ++i) { - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_2 = 0; i_2 < 8; ++i_2) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_4 = 0; i_4 < 8; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - tl::__sync_thread_partial<3, 64>(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)]; - #pragma unroll - for (int i_6 = 0; i_6 < 16; ++i_6) { - ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)])); - } - } - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - float4 __3; - float4 v__1 = *(float4*)(sumsq_per_pos + (i_7 * 4)); - float4 __4; - float4 v__2 = *(float4*)(ol + (i_7 * 4)); - __4.x = (v__2.x*v__2.x); - __4.y = (v__2.y*v__2.y); - __4.z = (v__2.z*v__2.z); - __4.w = (v__2.w*v__2.w); - __3.x = (v__1.x+__4.x); - __3.y = (v__1.y+__4.y); - __3.z = (v__1.z+__4.z); - __3.w = (v__1.w+__4.w); - *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3; - uint2 __5; - float4 v__3 = *(float4*)(ol + (i_7 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __5; - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_8 = 0; i_8 < 8; ++i_8) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __6; - uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]); - *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __6; - } - } - #pragma unroll - for (int i_9 = 0; i_9 < 4; ++i_9) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)]; - #pragma unroll - for (int i_10 = 0; i_10 < 16; ++i_10) { - ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)])); - } - } - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - float4 __7; - float4 v__5 = *(float4*)(sumsq_per_pos + (i_11 * 4)); - float4 __8; - float4 v__6 = *(float4*)(ol + (i_11 * 4)); - __8.x = (v__6.x*v__6.x); - __8.y = (v__6.y*v__6.y); - __8.z = (v__6.z*v__6.z); - __8.w = (v__6.w*v__6.w); - __7.x = (v__5.x+__8.x); - __7.y = (v__5.y+__8.y); - __7.z = (v__5.z+__8.z); - __7.w = (v__5.w+__8.w); - *(float4*)(sumsq_per_pos + (i_11 * 4)) = __7; - uint2 __9; - float4 v__7 = *(float4*)(ol + (i_11 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__9))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__9))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __9; - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_12 = 0; i_12 < 8; ++i_12) { - *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __10; - uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4)); - ((float2*)(&__10))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]); - ((float2*)(&__10))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]); - *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __10; - } - } - #pragma unroll - for (int i_13 = 0; i_13 < 4; ++i_13) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)]; - #pragma unroll - for (int i_14 = 0; i_14 < 16; ++i_14) { - ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)])); - } - } - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - float4 __11; - float4 v__9 = *(float4*)(sumsq_per_pos + (i_15 * 4)); - float4 __12; - float4 v__10 = *(float4*)(ol + (i_15 * 4)); - __12.x = (v__10.x*v__10.x); - __12.y = (v__10.y*v__10.y); - __12.z = (v__10.z*v__10.z); - __12.w = (v__10.w*v__10.w); - __11.x = (v__9.x+__12.x); - __11.y = (v__9.y+__12.y); - __11.z = (v__9.z+__12.z); - __11.w = (v__9.w+__12.w); - *(float4*)(sumsq_per_pos + (i_15 * 4)) = __11; - uint2 __13; - float4 v__11 = *(float4*)(ol + (i_15 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__13))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__13))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __13; - } - sumsq[0] = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int rv = 0; rv < 16; ++rv) { - sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]); - } - tl::__sync_thread_partial<3, 64>(); - sumsq[0] = tl::AllReduce>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192]))); - rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_17 = 0; i_17 < 2; ++i_17) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)]))); - } - tl::cp_async_commit(); - for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_18 = 0; i_18 < 2; ++i_18) { - *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_3 = 0; vec_3 < 2; ++vec_3) { - float4 __14; - uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4)); - ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]); - ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]); - *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __14; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_19 = 0; i_19 < 2; ++i_19) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)]))); - } - tl::cp_async_commit(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_20 = 0; i_20 < 2; ++i_20) { - *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984)); - for (int vec_4 = 0; vec_4 < 2; ++vec_4) { - float4 __15; - float4 __16; - float4 __17; - uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4)); - ((float2*)(&__17))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]); - ((float2*)(&__17))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]); - float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __16.x = (__17.x*v__14.x); - __16.y = (__17.y*v__14.y); - __16.z = (__17.z*v__14.z); - __16.w = (__17.w*v__14.w); - float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4))); - __15.x = (__16.x*v__15.x); - __15.y = (__16.y*v__15.y); - __15.z = (__16.z*v__15.z); - __15.w = (__16.w*v__15.w); - *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __15; - } - } - #pragma unroll - for (int i_21 = 0; i_21 < 2; ++i_21) { - for (int vec_5 = 0; vec_5 < 2; ++vec_5) { - uint2 __18; - float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__18))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__18))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]); - *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __18; - } - *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0); - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_22 = 0; i_22 < 2; ++i_22) { - *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_6 = 0; vec_6 < 2; ++vec_6) { - float4 __19; - uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4)); - ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]); - ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]); - *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __19; - } - } - #pragma unroll - for (int i_23 = 0; i_23 < 2; ++i_23) { - *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032)); - for (int vec_7 = 0; vec_7 < 2; ++vec_7) { - float4 __20; - float4 __21; - float4 __22; - uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4)); - ((float2*)(&__22))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]); - ((float2*)(&__22))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]); - float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __21.x = (__22.x*v__19.x); - __21.y = (__22.y*v__19.y); - __21.z = (__22.z*v__19.z); - __21.w = (__22.w*v__19.w); - float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4))); - __20.x = (__21.x*v__20.x); - __20.y = (__21.y*v__20.y); - __20.z = (__21.z*v__20.z); - __20.w = (__21.w*v__20.w); - *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __20; - } - } - #pragma unroll - for (int i_24 = 0; i_24 < 2; ++i_24) { - for (int vec_8 = 0; vec_8 < 2; ++vec_8) { - uint2 __23; - float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__23))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__23))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]); - *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __23; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0); - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_25 = 0; i_25 < 2; ++i_25) { - *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104)); - for (int vec_9 = 0; vec_9 < 2; ++vec_9) { - float4 __24; - uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4)); - ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]); - ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]); - *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __24; - } - } - #pragma unroll - for (int i_26 = 0; i_26 < 2; ++i_26) { - *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056)); - for (int vec_10 = 0; vec_10 < 2; ++vec_10) { - float4 __25; - float4 __26; - float4 __27; - uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4)); - ((float2*)(&__27))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]); - ((float2*)(&__27))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]); - float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __26.x = (__27.x*v__24.x); - __26.y = (__27.y*v__24.y); - __26.z = (__27.z*v__24.z); - __26.w = (__27.w*v__24.w); - float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4))); - __25.x = (__26.x*v__25.x); - __25.y = (__26.y*v__25.y); - __25.z = (__26.z*v__25.z); - __25.w = (__26.w*v__25.w); - *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __25; - } - } - #pragma unroll - for (int i_27 = 0; i_27 < 2; ++i_27) { - for (int vec_11 = 0; vec_11 < 2; ++vec_11) { - uint2 __28; - float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__28))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__28))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]); - *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __28; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/e8bfe015e13b0beed5fbf1df07913214e93cd08d927e1e810f022bb8d28739b0/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/e8bfe015e13b0beed5fbf1df07913214e93cd08d927e1e810f022bb8d28739b0/host_kernel.cu deleted file mode 100644 index 029d209..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/e8bfe015e13b0beed5fbf1df07913214e93cd08d927e1e810f022bb8d28739b0/host_kernel.cu +++ /dev/null @@ -1,3359 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_pre_big_fuse_with_norm_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[16]; - void* stack_ffi_any = stack; - if (!((num_args == 9))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_pre_big_fuse_with_norm_tilelang: num_args should be 9", (long long)(num_args), (long long)(9)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang: args pointer is NULL"); - return -1; - } - int32_t gemm_out_mul_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((gemm_out_mul_type_index == 0) || (gemm_out_mul_type_index == 4)) || (gemm_out_mul_type_index == 7)) || (64 <= gemm_out_mul_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_mul expected pointer or tensor handle"); - return -1; - } - int32_t gemm_out_sqrsum_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((gemm_out_sqrsum_type_index == 0) || (gemm_out_sqrsum_type_index == 4)) || (gemm_out_sqrsum_type_index == 7)) || (64 <= gemm_out_sqrsum_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_sqrsum expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t layer_input_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((layer_input_type_index == 0) || (layer_input_type_index == 4)) || (layer_input_type_index == 7)) || (64 <= layer_input_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input layer_input expected pointer or tensor handle"); - return -1; - } - int32_t norm_weight_type_index = (((TVMFFIAny*)args)[8].type_index); - if (!(((((norm_weight_type_index == 0) || (norm_weight_type_index == 4)) || (norm_weight_type_index == 7)) || (64 <= norm_weight_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input norm_weight expected pointer or tensor handle"); - return -1; - } - void* gemm_out_mul = ((gemm_out_mul_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* gemm_out_sqrsum = ((gemm_out_sqrsum_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* layer_input = ((layer_input_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - void* norm_weight = ((norm_weight_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[8].v_ptr) + 24)) : (((TVMFFIAny*)args)[8].v_ptr)); - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null = (gemm_out_mul == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null = (gemm_out_sqrsum == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_residual_is_null = (residual == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null = (layer_input == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.layer_input is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null = (norm_weight == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.norm_weight is expected to have non-NULL pointer"); - return -1; - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape = (((DLTensor*)gemm_out_mul)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape = (((DLTensor*)gemm_out_sqrsum)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape = (((DLTensor*)layer_input)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape = (((DLTensor*)norm_weight)[0].shape); - if ((((DLTensor*)gemm_out_mul)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul, mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum, mhc_pre_big_fuse_with_norm_tilelang.residual, mhc_pre_big_fuse_with_norm_tilelang.post_mix, mhc_pre_big_fuse_with_norm_tilelang.comb_mix, mhc_pre_big_fuse_with_norm_tilelang.layer_input"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides = (((DLTensor*)gemm_out_mul)[0].strides); - int32_t dev_id = (((DLTensor*)gemm_out_mul)[0].device.device_id); - void* gemm_out_mul_1 = (((DLTensor*)gemm_out_mul)[0].data); - if ((((DLTensor*)gemm_out_sqrsum)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides = (((DLTensor*)gemm_out_sqrsum)[0].strides); - void* gemm_out_sqrsum_1 = (((DLTensor*)gemm_out_sqrsum)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)comb_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)layer_input)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides = (((DLTensor*)layer_input)[0].strides); - void* layer_input_1 = (((DLTensor*)layer_input)[0].data); - if ((((DLTensor*)norm_weight)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_9) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides = (((DLTensor*)norm_weight)[0].strides); - void* norm_weight_1 = (((DLTensor*)norm_weight)[0].data); - if ((((((DLTensor*)gemm_out_mul)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_mul)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_mul)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[1]); - } - if ((condval_2 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - int32_t condval_3; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[0]); - } - if (((num_tokens * 24) != condval_3) && (num_tokens != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_15) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_mul)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_mul)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_17) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_18) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)gemm_out_sqrsum)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[1]); - } - if ((condval_4 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_22) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[0]); - } - if ((((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != condval_5) && (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_23) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_sqrsum)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_26) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_27) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_28) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_29) != 0) { - return -1; - } - } - int32_t condval_6; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - if (condval_6 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_7; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_7); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_34) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_35) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_36) != 0) { - return -1; - } - } - int32_t condval_8; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - if (condval_8 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_9; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_9); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_38) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_40) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_41) != 0) { - return -1; - } - } - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_42) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_46) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - if (condval_12 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_13; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_13); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_47) != 0) { - return -1; - } - } - int32_t condval_14; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[0]); - } - if ((condval_14 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_48) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_49) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_51) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_52) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_53) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_54) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - if (condval_15 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_16; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_16); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_56) != 0) { - return -1; - } - } - int32_t condval_17; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[0]); - } - if ((condval_17 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_58) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_60) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_61) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_62) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_63) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1]) != 16) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_64) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - if (condval_18 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[0]); - } - if ((condval_20 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_66) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_67) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_68) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_69) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_70) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)layer_input)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)layer_input)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)layer_input)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_71) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_72) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_73) != 0) { - return -1; - } - } - int32_t condval_21; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - if (condval_21 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_22; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_22); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - int32_t condval_23; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[0]); - } - if ((condval_23 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_75) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)layer_input)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_76) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_77) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_78) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_79) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)norm_weight)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)norm_weight)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)norm_weight)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_80) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - if (condval_24 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_25; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_25); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)norm_weight)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_83) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_85) != 0) { - return -1; - } - } - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_86) != 0) { - return -1; - } - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_87) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = gemm_out_mul_1; - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = gemm_out_sqrsum_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = hc_scale_1; - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = layer_input_1; - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = norm_weight_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = post_mix_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)96); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)29040); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = (int64_t)0; - if (mhc_pre_big_fuse_with_norm_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_pre_big_fuse_with_norm_tilelang_kernel", &mhc_pre_big_fuse_with_norm_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_pre_big_fuse_with_norm_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 15, &result_88) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/e8bfe015e13b0beed5fbf1df07913214e93cd08d927e1e810f022bb8d28739b0/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/e8bfe015e13b0beed5fbf1df07913214e93cd08d927e1e810f022bb8d28739b0/params.pkl deleted file mode 100644 index b0a972109daa58b487e1cc9fc85f613882b4fe51..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2200 zcmeHI&r2IY6jozmKq~<$YN24~6d^4BfF5%8HdMrlmm+IQBy; zV%04(1wWY%&!J-xTowwfGyzmz4BiO(a`#FtrcYih4(g->fU-FTi_56iXM{W@8z$%qzd<8cMb5s4Wa t5MBH>*O+=r&Dpc--K*kyebY_(;=g*|!{Uv4^;vmk3*YV*CgYU{!Cze&_6Gm} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/ee551f0fd55a5f344ee5ec2ee5aa79c7656eb4c9efbd0c4a7c2d3107c1e82478/device_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/ee551f0fd55a5f344ee5ec2ee5aa79c7656eb4c9efbd0c4a7c2d3107c1e82478/device_kernel.cu deleted file mode 100644 index 898a970..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/ee551f0fd55a5f344ee5ec2ee5aa79c7656eb4c9efbd0c4a7c2d3107c1e82478/device_kernel.cu +++ /dev/null @@ -1,424 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#ifdef ENABLE_BF16 -#include -#endif - -extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens); -extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) { - extern __shared__ __align__(1024) uchar buf_dyn_shmem[]; - float mixes[1]; - float rms[1]; - float cm[1]; - float row_max[1]; - float row_sum[1]; - float col_sum[1]; - float sumsq_per_pos[16]; - float sumsq[1]; - float rsqrt_norm[1]; - float xl[64]; - float ol[16]; - bfloat16_t xs_local_cast[8]; - bfloat16_t xs_local_cast_1[8]; - bfloat16_t xs_local_cast_2[8]; - float w_local[16]; - float ol_1[16]; - bfloat16_t w_shared_local_cast_3[8]; - bfloat16_t output_shared_local_cast_4[8]; - bfloat16_t layer_input_local_cast_5[8]; - bfloat16_t w_shared_local_cast_6[8]; - bfloat16_t output_shared_local_cast_7[8]; - bfloat16_t layer_input_local_cast_8[8]; - bfloat16_t w_shared_local_cast_9[8]; - bfloat16_t output_shared_local_cast_10[8]; - bfloat16_t layer_input_local_cast_11[8]; - mixes[0] = 0x0p+0f/*0.000000e+00*/; - rms[0] = 0x0p+0f/*0.000000e+00*/; - cudaGridDependencySynchronize(); - for (int i_split = 0; i_split < 7; ++i_split) { - rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]); - } - rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - mixes[0] = 0x0p+0f/*0.000000e+00*/; - for (int i_split_1 = 0; i_split_1 < 7; ++i_split_1) { - mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]); - } - mixes[0] = (mixes[0] * rms[0]); - if ((((int)threadIdx.x) / 24) == 0) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0]; - } - __syncthreads(); - if (((int)threadIdx.x) < 32) { - if (((int)threadIdx.x) < 4) { - post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/); - } - cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]); - row_max[0] = -CUDART_INF_F; - row_max[0] = max(row_max[0], cm[0]); - row_max[0] = tl::AllReduce>::run(row_max[0]); - cm[0] = expf((cm[0] - row_max[0])); - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - for (int __1 = 0; __1 < 19; ++__1) { - row_sum[0] = 0x0p+0f/*0.000000e+00*/; - row_sum[0] = (row_sum[0] + cm[0]); - row_sum[0] = tl::AllReduce>::run(row_sum[0]); - cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - col_sum[0] = 0x0p+0f/*0.000000e+00*/; - col_sum[0] = (col_sum[0] + cm[0]); - col_sum[0] = tl::AllReduce>::run(col_sum[0]); - cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - } - if ((((int)threadIdx.x) >> 4) == 0) { - comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0]; - } - } else { - if (((int)threadIdx.x) < 36) { - ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/); - } - #pragma unroll - for (int i = 0; i < 4; ++i) { - float broadcast_var = 0x0p+0f/*0.000000e+00*/; - *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var); - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_1 = 0; i_1 < 8; ++i_1) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_2 = 0; i_2 < 8; ++i_2) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)]))); - } - tl::cp_async_commit(); - for (int i0_h = 0; i0_h < 2; ++i0_h) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_3 = 0; i_3 < 8; ++i_3) { - *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec = 0; vec < 2; ++vec) { - float4 __2; - uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4)); - ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]); - ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]); - *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_4 = 0; i_4 < 8; ++i_4) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_5 = 0; i_5 < 4; ++i_5) { - float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1); - } - tl::__sync_thread_partial<3, 64>(); - for (int i_hc = 0; i_hc < 4; ++i_hc) { - float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)]; - #pragma unroll - for (int i_6 = 0; i_6 < 16; ++i_6) { - ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)])); - } - } - #pragma unroll - for (int i_7 = 0; i_7 < 4; ++i_7) { - float4 __3; - float4 v__1 = *(float4*)(sumsq_per_pos + (i_7 * 4)); - float4 __4; - float4 v__2 = *(float4*)(ol + (i_7 * 4)); - __4.x = (v__2.x*v__2.x); - __4.y = (v__2.y*v__2.y); - __4.z = (v__2.z*v__2.z); - __4.w = (v__2.w*v__2.w); - __3.x = (v__1.x+__4.x); - __3.y = (v__1.y+__4.y); - __3.z = (v__1.z+__4.z); - __3.w = (v__1.w+__4.w); - *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3; - uint2 __5; - float4 v__3 = *(float4*)(ol + (i_7 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__5))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__5))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __5; - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_8 = 0; i_8 < 8; ++i_8) { - *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208)); - for (int vec_1 = 0; vec_1 < 2; ++vec_1) { - float4 __6; - uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4)); - ((float2*)(&__6))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]); - ((float2*)(&__6))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]); - *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __6; - } - } - #pragma unroll - for (int i_9 = 0; i_9 < 4; ++i_9) { - float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2); - } - for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) { - float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)]; - #pragma unroll - for (int i_10 = 0; i_10 < 16; ++i_10) { - ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)])); - } - } - #pragma unroll - for (int i_11 = 0; i_11 < 4; ++i_11) { - float4 __7; - float4 v__5 = *(float4*)(sumsq_per_pos + (i_11 * 4)); - float4 __8; - float4 v__6 = *(float4*)(ol + (i_11 * 4)); - __8.x = (v__6.x*v__6.x); - __8.y = (v__6.y*v__6.y); - __8.z = (v__6.z*v__6.z); - __8.w = (v__6.w*v__6.w); - __7.x = (v__5.x+__8.x); - __7.y = (v__5.y+__8.y); - __7.z = (v__5.z+__8.z); - __7.w = (v__5.w+__8.w); - *(float4*)(sumsq_per_pos + (i_11 * 4)) = __7; - uint2 __9; - float4 v__7 = *(float4*)(ol + (i_11 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__9))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__9))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __9; - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_12 = 0; i_12 < 8; ++i_12) { - *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888)); - for (int vec_2 = 0; vec_2 < 2; ++vec_2) { - float4 __10; - uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4)); - ((float2*)(&__10))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]); - ((float2*)(&__10))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]); - *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __10; - } - } - #pragma unroll - for (int i_13 = 0; i_13 < 4; ++i_13) { - float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/; - *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3); - } - for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) { - float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)]; - #pragma unroll - for (int i_14 = 0; i_14 < 16; ++i_14) { - ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)])); - } - } - #pragma unroll - for (int i_15 = 0; i_15 < 4; ++i_15) { - float4 __11; - float4 v__9 = *(float4*)(sumsq_per_pos + (i_15 * 4)); - float4 __12; - float4 v__10 = *(float4*)(ol + (i_15 * 4)); - __12.x = (v__10.x*v__10.x); - __12.y = (v__10.y*v__10.y); - __12.z = (v__10.z*v__10.z); - __12.w = (v__10.w*v__10.w); - __11.x = (v__9.x+__12.x); - __11.y = (v__9.y+__12.y); - __11.z = (v__9.z+__12.z); - __11.w = (v__9.w+__12.w); - *(float4*)(sumsq_per_pos + (i_15 * 4)) = __11; - uint2 __13; - float4 v__11 = *(float4*)(ol + (i_15 * 4)); - (reinterpret_cast<__nv_bfloat162*>(&__13))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__13))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]); - *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __13; - } - sumsq[0] = 0x0p+0f/*0.000000e+00*/; - #pragma unroll - for (int rv = 0; rv < 16; ++rv) { - sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]); - } - tl::__sync_thread_partial<3, 64>(); - sumsq[0] = tl::AllReduce>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192]))); - rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/)); - #pragma unroll - for (int i_16 = 0; i_16 < 2; ++i_16) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)]))); - } - tl::cp_async_commit(); - #pragma unroll - for (int i_17 = 0; i_17 < 2; ++i_17) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)]))); - } - tl::cp_async_commit(); - for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) { - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_18 = 0; i_18 < 2; ++i_18) { - *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_3 = 0; vec_3 < 2; ++vec_3) { - float4 __14; - uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4)); - ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]); - ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]); - *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __14; - } - } - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_19 = 0; i_19 < 2; ++i_19) { - tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)]))); - } - tl::cp_async_commit(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_20 = 0; i_20 < 2; ++i_20) { - *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984)); - for (int vec_4 = 0; vec_4 < 2; ++vec_4) { - float4 __15; - float4 __16; - float4 __17; - uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4)); - ((float2*)(&__17))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]); - ((float2*)(&__17))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]); - float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __16.x = (__17.x*v__14.x); - __16.y = (__17.y*v__14.y); - __16.z = (__17.z*v__14.z); - __16.w = (__17.w*v__14.w); - float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4))); - __15.x = (__16.x*v__15.x); - __15.y = (__16.y*v__15.y); - __15.z = (__16.z*v__15.z); - __15.w = (__16.w*v__15.w); - *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __15; - } - } - #pragma unroll - for (int i_21 = 0; i_21 < 2; ++i_21) { - for (int vec_5 = 0; vec_5 < 2; ++vec_5) { - uint2 __18; - float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__18))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__18))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]); - *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __18; - } - *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0); - } - } - tl::cp_async_wait<1>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_22 = 0; i_22 < 2; ++i_22) { - *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080)); - for (int vec_6 = 0; vec_6 < 2; ++vec_6) { - float4 __19; - uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4)); - ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]); - ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]); - *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __19; - } - } - #pragma unroll - for (int i_23 = 0; i_23 < 2; ++i_23) { - *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032)); - for (int vec_7 = 0; vec_7 < 2; ++vec_7) { - float4 __20; - float4 __21; - float4 __22; - uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4)); - ((float2*)(&__22))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]); - ((float2*)(&__22))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]); - float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __21.x = (__22.x*v__19.x); - __21.y = (__22.y*v__19.y); - __21.z = (__22.z*v__19.z); - __21.w = (__22.w*v__19.w); - float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4))); - __20.x = (__21.x*v__20.x); - __20.y = (__21.y*v__20.y); - __20.z = (__21.z*v__20.z); - __20.w = (__21.w*v__20.w); - *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __20; - } - } - #pragma unroll - for (int i_24 = 0; i_24 < 2; ++i_24) { - for (int vec_8 = 0; vec_8 < 2; ++vec_8) { - uint2 __23; - float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__23))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__23))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]); - *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __23; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0); - } - tl::cp_async_wait<0>(); - tl::__sync_thread_partial<3, 64>(); - #pragma unroll - for (int i_25 = 0; i_25 < 2; ++i_25) { - *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104)); - for (int vec_9 = 0; vec_9 < 2; ++vec_9) { - float4 __24; - uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4)); - ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]); - ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]); - *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __24; - } - } - #pragma unroll - for (int i_26 = 0; i_26 < 2; ++i_26) { - *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056)); - for (int vec_10 = 0; vec_10 < 2; ++vec_10) { - float4 __25; - float4 __26; - float4 __27; - uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4)); - ((float2*)(&__27))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]); - ((float2*)(&__27))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]); - float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]); - __26.x = (__27.x*v__24.x); - __26.y = (__27.y*v__24.y); - __26.z = (__27.z*v__24.z); - __26.w = (__27.w*v__24.w); - float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4))); - __25.x = (__26.x*v__25.x); - __25.y = (__26.y*v__25.y); - __25.z = (__26.z*v__25.z); - __25.w = (__26.w*v__25.w); - *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __25; - } - } - #pragma unroll - for (int i_27 = 0; i_27 < 2; ++i_27) { - for (int vec_11 = 0; vec_11 < 2; ++vec_11) { - uint2 __28; - float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4))); - (reinterpret_cast<__nv_bfloat162*>(&__28))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]); - (reinterpret_cast<__nv_bfloat162*>(&__28))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]); - *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __28; - } - *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0); - } - } - cudaTriggerProgrammaticLaunchCompletion(); -} - diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/ee551f0fd55a5f344ee5ec2ee5aa79c7656eb4c9efbd0c4a7c2d3107c1e82478/host_kernel.cu b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/ee551f0fd55a5f344ee5ec2ee5aa79c7656eb4c9efbd0c4a7c2d3107c1e82478/host_kernel.cu deleted file mode 100644 index 7a8af95..0000000 --- a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/ee551f0fd55a5f344ee5ec2ee5aa79c7656eb4c9efbd0c4a7c2d3107c1e82478/host_kernel.cu +++ /dev/null @@ -1,3359 +0,0 @@ -// tilelang target: c -keys=cpu -#define TVM_EXPORTS -#include "tvm/runtime/base.h" -#include "tvm/runtime/c_backend_api.h" -#include "tvm/ffi/c_api.h" -#include -#include -#include -#ifdef __OBJC__ -#include "tvm/runtime/device_api.h" -#include "tvm/ffi/function.h" -#include -#include -#include -#endif -void* __tvm_ffi__library_ctx = NULL; -static void* __tvm_error_ndim_mismatch_packed = NULL; -static void* __tvm_error_dtype_mismatch_packed = NULL; -static void* __tvm_error_expect_eq_packed = NULL; -static void* __tvm_error_constraint_violation_packed = NULL; -static void* __tvm_error_byte_offset_mismatch_packed = NULL; -static void* __tvm_error_device_type_mismatch_packed = NULL; -static void* __tvm_error_null_ptr_packed = NULL; -static void* __tvm_set_device_packed = NULL; -static void* mhc_pre_big_fuse_with_norm_tilelang_kernel_packed = NULL; -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result); -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(void* self_handle, void* args, int32_t num_args, void* result) { - __attribute__((aligned(64))) TVMFFIAny stack[16]; - void* stack_ffi_any = stack; - if (!((num_args == 9))) { - char __tvm_assert_msg_buf[512]; - snprintf(__tvm_assert_msg_buf, 512, "%s; expected: %lld, got: %lld", "mhc_pre_big_fuse_with_norm_tilelang: num_args should be 9", (long long)(num_args), (long long)(9)); - TVMFFIErrorSetRaisedFromCStr("RuntimeError", __tvm_assert_msg_buf); - return -1; - } - if (!(!(args == NULL))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang: args pointer is NULL"); - return -1; - } - int32_t gemm_out_mul_type_index = (((TVMFFIAny*)args)[0].type_index); - if (!(((((gemm_out_mul_type_index == 0) || (gemm_out_mul_type_index == 4)) || (gemm_out_mul_type_index == 7)) || (64 <= gemm_out_mul_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_mul expected pointer or tensor handle"); - return -1; - } - int32_t gemm_out_sqrsum_type_index = (((TVMFFIAny*)args)[1].type_index); - if (!(((((gemm_out_sqrsum_type_index == 0) || (gemm_out_sqrsum_type_index == 4)) || (gemm_out_sqrsum_type_index == 7)) || (64 <= gemm_out_sqrsum_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input gemm_out_sqrsum expected pointer or tensor handle"); - return -1; - } - int32_t hc_scale_type_index = (((TVMFFIAny*)args)[2].type_index); - if (!(((((hc_scale_type_index == 0) || (hc_scale_type_index == 4)) || (hc_scale_type_index == 7)) || (64 <= hc_scale_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_scale expected pointer or tensor handle"); - return -1; - } - int32_t hc_base_type_index = (((TVMFFIAny*)args)[3].type_index); - if (!(((((hc_base_type_index == 0) || (hc_base_type_index == 4)) || (hc_base_type_index == 7)) || (64 <= hc_base_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input hc_base expected pointer or tensor handle"); - return -1; - } - int32_t residual_type_index = (((TVMFFIAny*)args)[4].type_index); - if (!(((((residual_type_index == 0) || (residual_type_index == 4)) || (residual_type_index == 7)) || (64 <= residual_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input residual expected pointer or tensor handle"); - return -1; - } - int32_t post_mix_type_index = (((TVMFFIAny*)args)[5].type_index); - if (!(((((post_mix_type_index == 0) || (post_mix_type_index == 4)) || (post_mix_type_index == 7)) || (64 <= post_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input post_mix expected pointer or tensor handle"); - return -1; - } - int32_t comb_mix_type_index = (((TVMFFIAny*)args)[6].type_index); - if (!(((((comb_mix_type_index == 0) || (comb_mix_type_index == 4)) || (comb_mix_type_index == 7)) || (64 <= comb_mix_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input comb_mix expected pointer or tensor handle"); - return -1; - } - int32_t layer_input_type_index = (((TVMFFIAny*)args)[7].type_index); - if (!(((((layer_input_type_index == 0) || (layer_input_type_index == 4)) || (layer_input_type_index == 7)) || (64 <= layer_input_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input layer_input expected pointer or tensor handle"); - return -1; - } - int32_t norm_weight_type_index = (((TVMFFIAny*)args)[8].type_index); - if (!(((((norm_weight_type_index == 0) || (norm_weight_type_index == 4)) || (norm_weight_type_index == 7)) || (64 <= norm_weight_type_index)))) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "kernel mhc_pre_big_fuse_with_norm_tilelang input norm_weight expected pointer or tensor handle"); - return -1; - } - void* gemm_out_mul = ((gemm_out_mul_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[0].v_ptr) + 24)) : (((TVMFFIAny*)args)[0].v_ptr)); - void* gemm_out_sqrsum = ((gemm_out_sqrsum_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[1].v_ptr) + 24)) : (((TVMFFIAny*)args)[1].v_ptr)); - void* hc_scale = ((hc_scale_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[2].v_ptr) + 24)) : (((TVMFFIAny*)args)[2].v_ptr)); - void* hc_base = ((hc_base_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[3].v_ptr) + 24)) : (((TVMFFIAny*)args)[3].v_ptr)); - void* residual = ((residual_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[4].v_ptr) + 24)) : (((TVMFFIAny*)args)[4].v_ptr)); - void* post_mix = ((post_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[5].v_ptr) + 24)) : (((TVMFFIAny*)args)[5].v_ptr)); - void* comb_mix = ((comb_mix_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[6].v_ptr) + 24)) : (((TVMFFIAny*)args)[6].v_ptr)); - void* layer_input = ((layer_input_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[7].v_ptr) + 24)) : (((TVMFFIAny*)args)[7].v_ptr)); - void* norm_weight = ((norm_weight_type_index == 70) ? ((void*)((char*)(((TVMFFIAny*)args)[8].v_ptr) + 24)) : (((TVMFFIAny*)args)[8].v_ptr)); - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null = (gemm_out_mul == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null = (gemm_out_sqrsum == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null = (hc_scale == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_scale_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_scale is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null = (hc_base == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_hc_base_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.hc_base is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_residual_is_null = (residual == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_residual_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.residual is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null = (post_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_post_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.post_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null = (comb_mix == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_comb_mix_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.comb_mix is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null = (layer_input == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_layer_input_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.layer_input is expected to have non-NULL pointer"); - return -1; - } - bool mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null = (norm_weight == NULL); - if (!(!mhc_pre_big_fuse_with_norm_tilelang_norm_weight_is_null)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "mhc_pre_big_fuse_with_norm_tilelang.norm_weight is expected to have non-NULL pointer"); - return -1; - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape = (((DLTensor*)gemm_out_mul)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape = (((DLTensor*)gemm_out_sqrsum)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape = (((DLTensor*)hc_scale)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape = (((DLTensor*)hc_base)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_residual_shape = (((DLTensor*)residual)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape = (((DLTensor*)post_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape = (((DLTensor*)comb_mix)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape = (((DLTensor*)layer_input)[0].shape); - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape = (((DLTensor*)norm_weight)[0].shape); - if ((((DLTensor*)gemm_out_mul)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_1; - result_1.type_index = kTVMFFINone; - result_1.zero_padding = 0; - result_1.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_1) != 0) { - return -1; - } - } - if (!((bool)1)) { - TVMFFIErrorSetRaisedFromCStr("RuntimeError", "Symbolic shape variable num_tokens requires at least one non-null buffer among: mhc_pre_big_fuse_with_norm_tilelang.gemm_out_mul, mhc_pre_big_fuse_with_norm_tilelang.gemm_out_sqrsum, mhc_pre_big_fuse_with_norm_tilelang.residual, mhc_pre_big_fuse_with_norm_tilelang.post_mix, mhc_pre_big_fuse_with_norm_tilelang.comb_mix, mhc_pre_big_fuse_with_norm_tilelang.layer_input"); - return -1; - } - int32_t num_tokens = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]); - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides = (((DLTensor*)gemm_out_mul)[0].strides); - int32_t dev_id = (((DLTensor*)gemm_out_mul)[0].device.device_id); - void* gemm_out_mul_1 = (((DLTensor*)gemm_out_mul)[0].data); - if ((((DLTensor*)gemm_out_sqrsum)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_2; - result_2.type_index = kTVMFFINone; - result_2.zero_padding = 0; - result_2.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_2) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides = (((DLTensor*)gemm_out_sqrsum)[0].strides); - void* gemm_out_sqrsum_1 = (((DLTensor*)gemm_out_sqrsum)[0].data); - if ((((DLTensor*)hc_scale)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_3; - result_3.type_index = kTVMFFINone; - result_3.zero_padding = 0; - result_3.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_3) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides = (((DLTensor*)hc_scale)[0].strides); - void* hc_scale_1 = (((DLTensor*)hc_scale)[0].data); - if ((((DLTensor*)hc_base)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_4; - result_4.type_index = kTVMFFINone; - result_4.zero_padding = 0; - result_4.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_4) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides = (((DLTensor*)hc_base)[0].strides); - void* hc_base_1 = (((DLTensor*)hc_base)[0].data); - if ((((DLTensor*)residual)[0].ndim) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_5; - result_5.type_index = kTVMFFINone; - result_5.zero_padding = 0; - result_5.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_5) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_residual_strides = (((DLTensor*)residual)[0].strides); - void* residual_1 = (((DLTensor*)residual)[0].data); - if ((((DLTensor*)post_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_6; - result_6.type_index = kTVMFFINone; - result_6.zero_padding = 0; - result_6.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_6) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides = (((DLTensor*)post_mix)[0].strides); - void* post_mix_1 = (((DLTensor*)post_mix)[0].data); - if ((((DLTensor*)comb_mix)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_7; - result_7.type_index = kTVMFFINone; - result_7.zero_padding = 0; - result_7.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_7) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides = (((DLTensor*)comb_mix)[0].strides); - void* comb_mix_1 = (((DLTensor*)comb_mix)[0].data); - if ((((DLTensor*)layer_input)[0].ndim) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_8; - result_8.type_index = kTVMFFINone; - result_8.zero_padding = 0; - result_8.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_8) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides = (((DLTensor*)layer_input)[0].strides); - void* layer_input_1 = (((DLTensor*)layer_input)[0].data); - if ((((DLTensor*)norm_weight)[0].ndim) != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].ndim)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_ndim_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_ndim_mismatch", &__tvm_error_ndim_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_9; - result_9.type_index = kTVMFFINone; - result_9.zero_padding = 0; - result_9.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_ndim_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_9) != 0) { - return -1; - } - } - void* mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides = (((DLTensor*)norm_weight)[0].strides); - void* norm_weight_1 = (((DLTensor*)norm_weight)[0].data); - if ((((((DLTensor*)gemm_out_mul)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_mul)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_mul)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_10; - result_10.type_index = kTVMFFINone; - result_10.zero_padding = 0; - result_10.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_10) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0]) != 7) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)7; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_11; - result_11.type_index = kTVMFFINone; - result_11.zero_padding = 0; - result_11.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_11) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_12; - result_12.type_index = kTVMFFINone; - result_12.zero_padding = 0; - result_12.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_12) != 0) { - return -1; - } - } - int32_t condval; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval = 1; - } else { - condval = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - if (condval != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_1; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_1 = 1; - } else { - condval_1 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_1); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_13; - result_13.type_index = kTVMFFINone; - result_13.zero_padding = 0; - result_13.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_13) != 0) { - return -1; - } - } - int32_t condval_2; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_2 = 1; - } else { - condval_2 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[1]); - } - if ((condval_2 != 24) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_14; - result_14.type_index = kTVMFFINone; - result_14.zero_padding = 0; - result_14.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_14) != 0) { - return -1; - } - } - int32_t condval_3; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides == NULL)) { - condval_3 = 1; - } else { - condval_3 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_strides)[0]); - } - if (((num_tokens * 24) != condval_3) && (num_tokens != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_15; - result_15.type_index = kTVMFFINone; - result_15.zero_padding = 0; - result_15.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_15) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_mul)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_16; - result_16.type_index = kTVMFFINone; - result_16.zero_padding = 0; - result_16.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_16) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_mul)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_17; - result_17.type_index = kTVMFFINone; - result_17.zero_padding = 0; - result_17.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_17) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_mul"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_18; - result_18.type_index = kTVMFFINone; - result_18.zero_padding = 0; - result_18.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_18) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)gemm_out_sqrsum)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_19; - result_19.type_index = kTVMFFINone; - result_19.zero_padding = 0; - result_19.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_19) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0]) != 7) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)7; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_20; - result_20.type_index = kTVMFFINone; - result_20.zero_padding = 0; - result_20.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_20) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_21; - result_21.type_index = kTVMFFINone; - result_21.zero_padding = 0; - result_21.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_21) != 0) { - return -1; - } - } - int32_t condval_4; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_4 = 1; - } else { - condval_4 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[1]); - } - if ((condval_4 != 1) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_22; - result_22.type_index = kTVMFFINone; - result_22.zero_padding = 0; - result_22.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_22) != 0) { - return -1; - } - } - int32_t condval_5; - if ((mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides == NULL)) { - condval_5 = 1; - } else { - condval_5 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_sqrsum_strides)[0]); - } - if ((((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != condval_5) && (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != 0)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_23; - result_23.type_index = kTVMFFINone; - result_23.zero_padding = 0; - result_23.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_23) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)gemm_out_sqrsum)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_24; - result_24.type_index = kTVMFFINone; - result_24.zero_padding = 0; - result_24.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_24) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_25; - result_25.type_index = kTVMFFINone; - result_25.zero_padding = 0; - result_25.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_25) != 0) { - return -1; - } - } - if ((((DLTensor*)gemm_out_sqrsum)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_sqrsum)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_26; - result_26.type_index = kTVMFFINone; - result_26.zero_padding = 0; - result_26.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_26) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"gemm_out_sqrsum"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_27; - result_27.type_index = kTVMFFINone; - result_27.zero_padding = 0; - result_27.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_27) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)hc_scale)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_scale)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_scale)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_28; - result_28.type_index = kTVMFFINone; - result_28.zero_padding = 0; - result_28.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_28) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0]) != 3) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)3; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_29; - result_29.type_index = kTVMFFINone; - result_29.zero_padding = 0; - result_29.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_29) != 0) { - return -1; - } - } - int32_t condval_6; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_6 = 1; - } else { - condval_6 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - if (condval_6 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_7; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides == NULL)) { - condval_7 = 1; - } else { - condval_7 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_scale_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_7); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_30; - result_30.type_index = kTVMFFINone; - result_30.zero_padding = 0; - result_30.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_30) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_scale)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_31; - result_31.type_index = kTVMFFINone; - result_31.zero_padding = 0; - result_31.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_31) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_32; - result_32.type_index = kTVMFFINone; - result_32.zero_padding = 0; - result_32.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_32) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_scale)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_scale)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_33; - result_33.type_index = kTVMFFINone; - result_33.zero_padding = 0; - result_33.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_33) != 0) { - return -1; - } - } - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_scale"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_34; - result_34.type_index = kTVMFFINone; - result_34.zero_padding = 0; - result_34.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_34) != 0) { - return -1; - } - } - if ((((((DLTensor*)hc_base)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)hc_base)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)hc_base)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_35; - result_35.type_index = kTVMFFINone; - result_35.zero_padding = 0; - result_35.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_35) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0]) != 24) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)24; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_36; - result_36.type_index = kTVMFFINone; - result_36.zero_padding = 0; - result_36.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_36) != 0) { - return -1; - } - } - int32_t condval_8; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_8 = 1; - } else { - condval_8 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - if (condval_8 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_9; - if ((mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides == NULL)) { - condval_9 = 1; - } else { - condval_9 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_hc_base_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_9); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_37; - result_37.type_index = kTVMFFINone; - result_37.zero_padding = 0; - result_37.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_37) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)hc_base)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_38; - result_38.type_index = kTVMFFINone; - result_38.zero_padding = 0; - result_38.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_38) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_39; - result_39.type_index = kTVMFFINone; - result_39.zero_padding = 0; - result_39.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_39) != 0) { - return -1; - } - } - if ((((DLTensor*)hc_base)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)hc_base)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_40; - result_40.type_index = kTVMFFINone; - result_40.zero_padding = 0; - result_40.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_40) != 0) { - return -1; - } - } - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"hc_base"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_41; - result_41.type_index = kTVMFFINone; - result_41.zero_padding = 0; - result_41.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_41) != 0) { - return -1; - } - } - if ((((((DLTensor*)residual)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)residual)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)residual)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_42; - result_42.type_index = kTVMFFINone; - result_42.zero_padding = 0; - result_42.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_42) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_43; - result_43.type_index = kTVMFFINone; - result_43.zero_padding = 0; - result_43.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_43) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_44; - result_44.type_index = kTVMFFINone; - result_44.zero_padding = 0; - result_44.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_44) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_shape)[2])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_45; - result_45.type_index = kTVMFFINone; - result_45.zero_padding = 0; - result_45.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_45) != 0) { - return -1; - } - } - int32_t condval_10; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_10 = 1; - } else { - condval_10 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - if (condval_10 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[2]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_11; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_11 = 1; - } else { - condval_11 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[2]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_11); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_46; - result_46.type_index = kTVMFFINone; - result_46.zero_padding = 0; - result_46.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_46) != 0) { - return -1; - } - } - int32_t condval_12; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_12 = 1; - } else { - condval_12 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - if (condval_12 != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_13; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_13 = 1; - } else { - condval_13 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_13); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_47; - result_47.type_index = kTVMFFINone; - result_47.zero_padding = 0; - result_47.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_47) != 0) { - return -1; - } - } - int32_t condval_14; - if ((mhc_pre_big_fuse_with_norm_tilelang_residual_strides == NULL)) { - condval_14 = 1; - } else { - condval_14 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_residual_strides)[0]); - } - if ((condval_14 != 16384) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_48; - result_48.type_index = kTVMFFINone; - result_48.zero_padding = 0; - result_48.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_48) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)residual)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_49; - result_49.type_index = kTVMFFINone; - result_49.zero_padding = 0; - result_49.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_49) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_50; - result_50.type_index = kTVMFFINone; - result_50.zero_padding = 0; - result_50.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_50) != 0) { - return -1; - } - } - if ((((DLTensor*)residual)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)residual)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_51; - result_51.type_index = kTVMFFINone; - result_51.zero_padding = 0; - result_51.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_51) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"residual"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_52; - result_52.type_index = kTVMFFINone; - result_52.zero_padding = 0; - result_52.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_52) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)post_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)post_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)post_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_53; - result_53.type_index = kTVMFFINone; - result_53.zero_padding = 0; - result_53.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_53) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_54; - result_54.type_index = kTVMFFINone; - result_54.zero_padding = 0; - result_54.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_54) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1]) != 4) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_55; - result_55.type_index = kTVMFFINone; - result_55.zero_padding = 0; - result_55.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_55) != 0) { - return -1; - } - } - int32_t condval_15; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_15 = 1; - } else { - condval_15 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - if (condval_15 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_16; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_16 = 1; - } else { - condval_16 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_16); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_56; - result_56.type_index = kTVMFFINone; - result_56.zero_padding = 0; - result_56.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_56) != 0) { - return -1; - } - } - int32_t condval_17; - if ((mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides == NULL)) { - condval_17 = 1; - } else { - condval_17 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_post_mix_strides)[0]); - } - if ((condval_17 != 4) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_57; - result_57.type_index = kTVMFFINone; - result_57.zero_padding = 0; - result_57.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_57) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)post_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_58; - result_58.type_index = kTVMFFINone; - result_58.zero_padding = 0; - result_58.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_58) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_59; - result_59.type_index = kTVMFFINone; - result_59.zero_padding = 0; - result_59.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_59) != 0) { - return -1; - } - } - if ((((DLTensor*)post_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)post_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_60; - result_60.type_index = kTVMFFINone; - result_60.zero_padding = 0; - result_60.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_60) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"post_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_61; - result_61.type_index = kTVMFFINone; - result_61.zero_padding = 0; - result_61.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_61) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)comb_mix)[0].dtype.code) != (uint8_t)2) || ((((DLTensor*)comb_mix)[0].dtype.bits) != (uint8_t)32)) || ((((DLTensor*)comb_mix)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)32; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_62; - result_62.type_index = kTVMFFINone; - result_62.zero_padding = 0; - result_62.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_62) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_63; - result_63.type_index = kTVMFFINone; - result_63.zero_padding = 0; - result_63.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_63) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1]) != 16) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_64; - result_64.type_index = kTVMFFINone; - result_64.zero_padding = 0; - result_64.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_64) != 0) { - return -1; - } - } - int32_t condval_18; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_18 = 1; - } else { - condval_18 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - if (condval_18 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_19; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_19 = 1; - } else { - condval_19 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_19); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_65; - result_65.type_index = kTVMFFINone; - result_65.zero_padding = 0; - result_65.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_65) != 0) { - return -1; - } - } - int32_t condval_20; - if ((mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides == NULL)) { - condval_20 = 1; - } else { - condval_20 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_comb_mix_strides)[0]); - } - if ((condval_20 != 16) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_66; - result_66.type_index = kTVMFFINone; - result_66.zero_padding = 0; - result_66.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_66) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)comb_mix)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_67; - result_67.type_index = kTVMFFINone; - result_67.zero_padding = 0; - result_67.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_67) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_68; - result_68.type_index = kTVMFFINone; - result_68.zero_padding = 0; - result_68.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_68) != 0) { - return -1; - } - } - if ((((DLTensor*)comb_mix)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)comb_mix)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_69; - result_69.type_index = kTVMFFINone; - result_69.zero_padding = 0; - result_69.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_69) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"comb_mix"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_70; - result_70.type_index = kTVMFFINone; - result_70.zero_padding = 0; - result_70.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_70) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)layer_input)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)layer_input)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)layer_input)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_71; - result_71.type_index = kTVMFFINone; - result_71.zero_padding = 0; - result_71.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_71) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1]) != ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_gemm_out_mul_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_72; - result_72.type_index = kTVMFFINone; - result_72.zero_padding = 0; - result_72.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_72) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_shape)[1])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_73; - result_73.type_index = kTVMFFINone; - result_73.zero_padding = 0; - result_73.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_73) != 0) { - return -1; - } - } - int32_t condval_21; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_21 = 1; - } else { - condval_21 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - if (condval_21 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[1]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_22; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_22 = 1; - } else { - condval_22 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[1]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_22); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_74; - result_74.type_index = kTVMFFINone; - result_74.zero_padding = 0; - result_74.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_74) != 0) { - return -1; - } - } - int32_t condval_23; - if ((mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides == NULL)) { - condval_23 = 1; - } else { - condval_23 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_layer_input_strides)[0]); - } - if ((condval_23 != 4096) && (num_tokens != 1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_constraint_violation_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_constraint_violation", &__tvm_error_constraint_violation_packed) != 0) { - return -1; - } - } - TVMFFIAny result_75; - result_75.type_index = kTVMFFINone; - result_75.zero_padding = 0; - result_75.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_constraint_violation_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_75) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)layer_input)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_76; - result_76.type_index = kTVMFFINone; - result_76.zero_padding = 0; - result_76.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_76) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_77; - result_77.type_index = kTVMFFINone; - result_77.zero_padding = 0; - result_77.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_77) != 0) { - return -1; - } - } - if ((((DLTensor*)layer_input)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)layer_input)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_78; - result_78.type_index = kTVMFFINone; - result_78.zero_padding = 0; - result_78.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_78) != 0) { - return -1; - } - } - if (num_tokens != 0) { - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"layer_input"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_79; - result_79.type_index = kTVMFFINone; - result_79.zero_padding = 0; - result_79.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_79) != 0) { - return -1; - } - } - } else { - } - if ((((((DLTensor*)norm_weight)[0].dtype.code) != (uint8_t)4) || ((((DLTensor*)norm_weight)[0].dtype.bits) != (uint8_t)16)) || ((((DLTensor*)norm_weight)[0].dtype.lanes) != (uint16_t)1)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.code)); - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.bits)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].dtype.lanes)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)4; - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = (int64_t)16; - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = (int64_t)0; - if (__tvm_error_dtype_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_dtype_mismatch", &__tvm_error_dtype_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_80; - result_80.type_index = kTVMFFINone; - result_80.zero_padding = 0; - result_80.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_dtype_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 8, &result_80) != 0) { - return -1; - } - } - if (((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0]) != 4096) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"shape[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)4096; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_shape)[0])); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_81; - result_81.type_index = kTVMFFINone; - result_81.zero_padding = 0; - result_81.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_81) != 0) { - return -1; - } - } - int32_t condval_24; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_24 = 1; - } else { - condval_24 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - if (condval_24 != 1) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"strides[0]"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)1; - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - int32_t condval_25; - if ((mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides == NULL)) { - condval_25 = 1; - } else { - condval_25 = ((int32_t)((int64_t*)mhc_pre_big_fuse_with_norm_tilelang_norm_weight_strides)[0]); - } - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)condval_25); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_82; - result_82.type_index = kTVMFFINone; - result_82.zero_padding = 0; - result_82.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_82) != 0) { - return -1; - } - } - if ((uint64_t)0 != (((DLTensor*)norm_weight)[0].byte_offset)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].byte_offset)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_byte_offset_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_byte_offset_mismatch", &__tvm_error_byte_offset_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_83; - result_83.type_index = kTVMFFINone; - result_83.zero_padding = 0; - result_83.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_byte_offset_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_83) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_id) != (((DLTensor*)gemm_out_mul)[0].device.device_id)) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"device_id"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)gemm_out_mul)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_id)); - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = (int64_t)0; - if (__tvm_error_expect_eq_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_expect_eq", &__tvm_error_expect_eq_packed) != 0) { - return -1; - } - } - TVMFFIAny result_84; - result_84.type_index = kTVMFFINone; - result_84.zero_padding = 0; - result_84.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_expect_eq_packed, (TVMFFIAny*) stack_ffi_any, 5, &result_84) != 0) { - return -1; - } - } - if ((((DLTensor*)norm_weight)[0].device.device_type) != 2) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)2; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = ((int64_t)(((DLTensor*)norm_weight)[0].device.device_type)); - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = (int64_t)0; - if (__tvm_error_device_type_mismatch_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_device_type_mismatch", &__tvm_error_device_type_mismatch_packed) != 0) { - return -1; - } - } - TVMFFIAny result_85; - result_85.type_index = kTVMFFINone; - result_85.zero_padding = 0; - result_85.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_device_type_mismatch_packed, (TVMFFIAny*) stack_ffi_any, 4, &result_85) != 0) { - return -1; - } - } - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = (void*)"mhc_pre_big_fuse_with_norm_tilelang"; - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = (void*)"norm_weight"; - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 8; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = (void*)"data pointer"; - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = (int64_t)0; - if (__tvm_error_null_ptr_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_error_null_ptr", &__tvm_error_null_ptr_packed) != 0) { - return -1; - } - } - TVMFFIAny result_86; - result_86.type_index = kTVMFFINone; - result_86.zero_padding = 0; - result_86.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_error_null_ptr_packed, (TVMFFIAny*) stack_ffi_any, 3, &result_86) != 0) { - return -1; - } - } - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = ((int64_t)2); - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = ((int64_t)dev_id); - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = (int64_t)0; - if (__tvm_set_device_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "__tvm_set_device", &__tvm_set_device_packed) != 0) { - return -1; - } - } - TVMFFIAny result_87; - result_87.type_index = kTVMFFINone; - result_87.zero_padding = 0; - result_87.v_int64 = 0; - if (TVMFFIFunctionCall(__tvm_set_device_packed, (TVMFFIAny*) stack_ffi_any, 2, &result_87) != 0) { - return -1; - } - if (comb_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[0].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[0].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[0].v_ptr) = comb_mix_1; - if (gemm_out_mul_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[1].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[1].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[1].v_ptr) = gemm_out_mul_1; - if (gemm_out_sqrsum_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[2].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[2].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[2].v_ptr) = gemm_out_sqrsum_1; - if (hc_base_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[3].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[3].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[3].v_ptr) = hc_base_1; - if (hc_scale_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[4].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[4].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[4].v_ptr) = hc_scale_1; - if (layer_input_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[5].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[5].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[5].v_ptr) = layer_input_1; - if (norm_weight_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[6].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[6].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[6].v_ptr) = norm_weight_1; - if (post_mix_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[7].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[7].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[7].v_ptr) = post_mix_1; - if (residual_1 == NULL) { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 0; - } else { - (((TVMFFIAny*)stack_ffi_any)[8].type_index) = 4; - } - (((TVMFFIAny*)stack_ffi_any)[8].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_int64) = 0; - (((TVMFFIAny*)stack_ffi_any)[8].v_ptr) = residual_1; - (((TVMFFIAny*)stack_ffi_any)[9].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[9].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[9].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[10].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[10].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[10].v_int64) = ((int64_t)num_tokens); - (((TVMFFIAny*)stack_ffi_any)[11].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[11].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[11].v_int64) = ((int64_t)96); - (((TVMFFIAny*)stack_ffi_any)[12].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[12].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[12].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[13].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[13].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[13].v_int64) = ((int64_t)1); - (((TVMFFIAny*)stack_ffi_any)[14].type_index) = 1; - (((TVMFFIAny*)stack_ffi_any)[14].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[14].v_int64) = ((int64_t)29040); - (((TVMFFIAny*)stack_ffi_any)[15].type_index) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].zero_padding) = 0; - (((TVMFFIAny*)stack_ffi_any)[15].v_int64) = (int64_t)0; - if (mhc_pre_big_fuse_with_norm_tilelang_kernel_packed == NULL) { - if (TVMBackendGetFuncFromEnv(__tvm_ffi__library_ctx, "mhc_pre_big_fuse_with_norm_tilelang_kernel", &mhc_pre_big_fuse_with_norm_tilelang_kernel_packed) != 0) { - return -1; - } - } - TVMFFIAny result_88; - result_88.type_index = kTVMFFINone; - result_88.zero_padding = 0; - result_88.v_int64 = 0; - if (TVMFFIFunctionCall(mhc_pre_big_fuse_with_norm_tilelang_kernel_packed, (TVMFFIAny*) stack_ffi_any, 15, &result_88) != 0) { - return -1; - } - return 0; -} - -// CodegenC: NOTE: Auto-generated entry function -#ifdef __cplusplus -extern "C" -#endif -int32_t __tvm_ffi_main(void* self, void* args,int num_args, void* result) { - return __tvm_ffi_mhc_pre_big_fuse_with_norm_tilelang(self, args, num_args, result); -} diff --git a/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/ee551f0fd55a5f344ee5ec2ee5aa79c7656eb4c9efbd0c4a7c2d3107c1e82478/params.pkl b/experiments/h20/dsv4_dspark_h20_vllm_tp_dp_matrix/runtime/cache/tilelang/cache/0.1.9-x86_64/kernels/ee551f0fd55a5f344ee5ec2ee5aa79c7656eb4c9efbd0c4a7c2d3107c1e82478/params.pkl deleted file mode 100644 index 456ab98c4fc4438018af8288b4357746957bb88c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2200 zcmeHIO-my|5Y>%|0Y3;VvMMZSpRy2!RbdY~@2ycpm%S{~^kgdO-ks?ldOE917+4Q_ zYENl@pMS-gN#ZvsxyHG5@v7ddS5w{ptJhccx$-;lO}A!c3^7-|08~#Z2vTl%;@uJj zqaZ#kPecFZ+~yZ*wT`-2=&Yz!Nt-0~!? zqYYNsU>yV80_&7jSl8%;*?t;>A0pWkbp1QG{6|4Q^z;MWqVv`t`Rvk%u9^vsaqNdK zidDD76r3_0{)Uc4a9Jp@(gbjp8t*DUxxw%SW~0cO;3`5!cQYVHYs;jH;Ec5QT14px zGSdDvAVt@tO%AjIlDF>8g#*XdNYx`P5}`cAs6O_f|A5LKB9Z*i$fS^e2OW?s<%+bw zV(yqgD)f)!4=4)RsQ2|Gl*8+6rV??iS1c%Yih4(g->fUU4NQd56iXM`8HXr$%qzc<9-Fo5s4Wa t5M9`xYfL?*=Iq(^?p1NUzUiiX_Fuj4Vev-2_^iCJg>QEYlkv)f;1-p6_74C6