| 1 | //===- NVVMToLLVMIRTranslation.cpp - Translate NVVM to LLVM IR ------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements a translation between the MLIR NVVM dialect and |
| 10 | // LLVM IR. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.h" |
| 15 | #include "mlir/Dialect/LLVMIR/LLVMModuleOps.h" |
| 16 | #include "mlir/Dialect/LLVMIR/NVVMDialect.h" |
| 17 | #include "mlir/IR/Operation.h" |
| 18 | #include "mlir/Target/LLVMIR/ModuleTranslation.h" |
| 19 | |
| 20 | #include "llvm/ADT/StringExtras.h" |
| 21 | #include "llvm/ADT/iterator_range.h" |
| 22 | #include "llvm/IR/IRBuilder.h" |
| 23 | #include "llvm/IR/IntrinsicsNVPTX.h" |
| 24 | #include "llvm/Support/FormatVariadic.h" |
| 25 | #include "llvm/Support/NVVMAttributes.h" |
| 26 | #include <cmath> |
| 27 | |
| 28 | using namespace mlir; |
| 29 | using namespace mlir::LLVM; |
| 30 | using mlir::LLVM::detail::createIntrinsicCall; |
| 31 | |
| 32 | #define REDUX_F32_ID_IMPL(op, abs, hasNaN) \ |
| 33 | hasNaN ? llvm::Intrinsic::nvvm_redux_sync_f##op##abs##_NaN \ |
| 34 | : llvm::Intrinsic::nvvm_redux_sync_f##op##abs |
| 35 | |
| 36 | #define GET_REDUX_F32_ID(op, hasAbs, hasNaN) \ |
| 37 | hasAbs ? REDUX_F32_ID_IMPL(op, _abs, hasNaN) : REDUX_F32_ID_IMPL(op, , hasNaN) |
| 38 | |
| 39 | static llvm::Intrinsic::ID getReduxIntrinsicId(llvm::Type *resultType, |
| 40 | NVVM::ReductionKind kind, |
| 41 | bool hasAbs, bool hasNaN) { |
| 42 | switch (kind) { |
| 43 | case NVVM::ReductionKind::ADD: |
| 44 | return llvm::Intrinsic::nvvm_redux_sync_add; |
| 45 | case NVVM::ReductionKind::UMAX: |
| 46 | return llvm::Intrinsic::nvvm_redux_sync_umax; |
| 47 | case NVVM::ReductionKind::UMIN: |
| 48 | return llvm::Intrinsic::nvvm_redux_sync_umin; |
| 49 | case NVVM::ReductionKind::AND: |
| 50 | return llvm::Intrinsic::nvvm_redux_sync_and; |
| 51 | case NVVM::ReductionKind::OR: |
| 52 | return llvm::Intrinsic::nvvm_redux_sync_or; |
| 53 | case NVVM::ReductionKind::XOR: |
| 54 | return llvm::Intrinsic::nvvm_redux_sync_xor; |
| 55 | case NVVM::ReductionKind::MAX: |
| 56 | return llvm::Intrinsic::nvvm_redux_sync_max; |
| 57 | case NVVM::ReductionKind::MIN: |
| 58 | return llvm::Intrinsic::nvvm_redux_sync_min; |
| 59 | case NVVM::ReductionKind::FMIN: |
| 60 | return GET_REDUX_F32_ID(min, hasAbs, hasNaN); |
| 61 | case NVVM::ReductionKind::FMAX: |
| 62 | return GET_REDUX_F32_ID(max, hasAbs, hasNaN); |
| 63 | } |
| 64 | llvm_unreachable("unknown reduction kind" ); |
| 65 | } |
| 66 | |
| 67 | static llvm::Intrinsic::ID getShflIntrinsicId(llvm::Type *resultType, |
| 68 | NVVM::ShflKind kind, |
| 69 | bool withPredicate) { |
| 70 | |
| 71 | if (withPredicate) { |
| 72 | resultType = cast<llvm::StructType>(Val: resultType)->getElementType(N: 0); |
| 73 | switch (kind) { |
| 74 | case NVVM::ShflKind::bfly: |
| 75 | return resultType->isFloatTy() |
| 76 | ? llvm::Intrinsic::nvvm_shfl_sync_bfly_f32p |
| 77 | : llvm::Intrinsic::nvvm_shfl_sync_bfly_i32p; |
| 78 | case NVVM::ShflKind::up: |
| 79 | return resultType->isFloatTy() ? llvm::Intrinsic::nvvm_shfl_sync_up_f32p |
| 80 | : llvm::Intrinsic::nvvm_shfl_sync_up_i32p; |
| 81 | case NVVM::ShflKind::down: |
| 82 | return resultType->isFloatTy() |
| 83 | ? llvm::Intrinsic::nvvm_shfl_sync_down_f32p |
| 84 | : llvm::Intrinsic::nvvm_shfl_sync_down_i32p; |
| 85 | case NVVM::ShflKind::idx: |
| 86 | return resultType->isFloatTy() ? llvm::Intrinsic::nvvm_shfl_sync_idx_f32p |
| 87 | : llvm::Intrinsic::nvvm_shfl_sync_idx_i32p; |
| 88 | } |
| 89 | } else { |
| 90 | switch (kind) { |
| 91 | case NVVM::ShflKind::bfly: |
| 92 | return resultType->isFloatTy() ? llvm::Intrinsic::nvvm_shfl_sync_bfly_f32 |
| 93 | : llvm::Intrinsic::nvvm_shfl_sync_bfly_i32; |
| 94 | case NVVM::ShflKind::up: |
| 95 | return resultType->isFloatTy() ? llvm::Intrinsic::nvvm_shfl_sync_up_f32 |
| 96 | : llvm::Intrinsic::nvvm_shfl_sync_up_i32; |
| 97 | case NVVM::ShflKind::down: |
| 98 | return resultType->isFloatTy() ? llvm::Intrinsic::nvvm_shfl_sync_down_f32 |
| 99 | : llvm::Intrinsic::nvvm_shfl_sync_down_i32; |
| 100 | case NVVM::ShflKind::idx: |
| 101 | return resultType->isFloatTy() ? llvm::Intrinsic::nvvm_shfl_sync_idx_f32 |
| 102 | : llvm::Intrinsic::nvvm_shfl_sync_idx_i32; |
| 103 | } |
| 104 | } |
| 105 | llvm_unreachable("unknown shuffle kind" ); |
| 106 | } |
| 107 | |
| 108 | static llvm::Intrinsic::ID getMatchSyncIntrinsicId(Type valType, |
| 109 | NVVM::MatchSyncKind kind) { |
| 110 | switch (kind) { |
| 111 | case NVVM::MatchSyncKind::any: |
| 112 | return valType.isInteger(width: 32) ? llvm::Intrinsic::nvvm_match_any_sync_i32 |
| 113 | : llvm::Intrinsic::nvvm_match_any_sync_i64; |
| 114 | case NVVM::MatchSyncKind::all: |
| 115 | // match.all instruction has two variants -- one returns a single value, |
| 116 | // another returns a pair {value, predicate}. We currently only implement |
| 117 | // the latter as that's the variant exposed by CUDA API. |
| 118 | return valType.isInteger(width: 32) ? llvm::Intrinsic::nvvm_match_all_sync_i32p |
| 119 | : llvm::Intrinsic::nvvm_match_all_sync_i64p; |
| 120 | } |
| 121 | llvm_unreachable("unsupported match sync kind" ); |
| 122 | } |
| 123 | |
| 124 | static llvm::Intrinsic::ID getVoteSyncIntrinsicId(NVVM::VoteSyncKind kind) { |
| 125 | switch (kind) { |
| 126 | case NVVM::VoteSyncKind::any: |
| 127 | return llvm::Intrinsic::nvvm_vote_any_sync; |
| 128 | case NVVM::VoteSyncKind::all: |
| 129 | return llvm::Intrinsic::nvvm_vote_all_sync; |
| 130 | case NVVM::VoteSyncKind::ballot: |
| 131 | return llvm::Intrinsic::nvvm_vote_ballot_sync; |
| 132 | case NVVM::VoteSyncKind::uni: |
| 133 | return llvm::Intrinsic::nvvm_vote_uni_sync; |
| 134 | } |
| 135 | llvm_unreachable("unsupported vote kind" ); |
| 136 | } |
| 137 | |
| 138 | static llvm::Intrinsic::ID |
| 139 | getLdMatrixIntrinsicId(NVVM::MMALayout layout, int32_t num, |
| 140 | NVVM::LdStMatrixShapeAttr shape, |
| 141 | NVVM::LdStMatrixEltType eltType) { |
| 142 | if (shape.getM() == 8 && shape.getN() == 8) { |
| 143 | switch (num) { |
| 144 | case 1: |
| 145 | return (layout == NVVM::MMALayout::row) |
| 146 | ? llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_b16 |
| 147 | : llvm::Intrinsic:: |
| 148 | nvvm_ldmatrix_sync_aligned_m8n8_x1_trans_b16; |
| 149 | case 2: |
| 150 | return (layout == NVVM::MMALayout::row) |
| 151 | ? llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_b16 |
| 152 | : llvm::Intrinsic:: |
| 153 | nvvm_ldmatrix_sync_aligned_m8n8_x2_trans_b16; |
| 154 | case 4: |
| 155 | return (layout == NVVM::MMALayout::row) |
| 156 | ? llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_b16 |
| 157 | : llvm::Intrinsic:: |
| 158 | nvvm_ldmatrix_sync_aligned_m8n8_x4_trans_b16; |
| 159 | } |
| 160 | } else if (shape.getM() == 8 && shape.getN() == 16) { |
| 161 | if (eltType == NVVM::LdStMatrixEltType::B8X16_B6X16_P32) { |
| 162 | switch (num) { |
| 163 | case 1: |
| 164 | return llvm::Intrinsic:: |
| 165 | nvvm_ldmatrix_sync_aligned_m8n16_x1_b8x16_b6x16_p32; |
| 166 | case 2: |
| 167 | return llvm::Intrinsic:: |
| 168 | nvvm_ldmatrix_sync_aligned_m8n16_x2_b8x16_b6x16_p32; |
| 169 | case 4: |
| 170 | return llvm::Intrinsic:: |
| 171 | nvvm_ldmatrix_sync_aligned_m8n16_x4_b8x16_b6x16_p32; |
| 172 | } |
| 173 | } else if (eltType == NVVM::LdStMatrixEltType::B8X16_B4X16_P64) { |
| 174 | switch (num) { |
| 175 | case 1: |
| 176 | return llvm::Intrinsic:: |
| 177 | nvvm_ldmatrix_sync_aligned_m8n16_x1_b8x16_b4x16_p64; |
| 178 | case 2: |
| 179 | return llvm::Intrinsic:: |
| 180 | nvvm_ldmatrix_sync_aligned_m8n16_x2_b8x16_b4x16_p64; |
| 181 | case 4: |
| 182 | return llvm::Intrinsic:: |
| 183 | nvvm_ldmatrix_sync_aligned_m8n16_x4_b8x16_b4x16_p64; |
| 184 | } |
| 185 | } else if (eltType == NVVM::LdStMatrixEltType::S8_S4) { |
| 186 | switch (num) { |
| 187 | case 1: |
| 188 | return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x1_s8_s4; |
| 189 | case 2: |
| 190 | return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x2_s8_s4; |
| 191 | case 4: |
| 192 | return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x4_s8_s4; |
| 193 | } |
| 194 | } |
| 195 | } else if (shape.getM() == 16 && shape.getN() == 16) { |
| 196 | if (eltType == NVVM::LdStMatrixEltType::B8) { |
| 197 | switch (num) { |
| 198 | case 1: |
| 199 | return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8; |
| 200 | case 2: |
| 201 | return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8; |
| 202 | } |
| 203 | } else if (eltType == NVVM::LdStMatrixEltType::B8X16_B6X16_P32) { |
| 204 | switch (num) { |
| 205 | case 1: |
| 206 | return llvm::Intrinsic:: |
| 207 | nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8x16_b6x16_p32; |
| 208 | case 2: |
| 209 | return llvm::Intrinsic:: |
| 210 | nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8x16_b6x16_p32; |
| 211 | } |
| 212 | } else if (eltType == NVVM::LdStMatrixEltType::B8X16_B4X16_P64) { |
| 213 | switch (num) { |
| 214 | case 1: |
| 215 | return llvm::Intrinsic:: |
| 216 | nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8x16_b4x16_p64; |
| 217 | case 2: |
| 218 | return llvm::Intrinsic:: |
| 219 | nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8x16_b4x16_p64; |
| 220 | } |
| 221 | } |
| 222 | } |
| 223 | llvm_unreachable("unknown ldmatrix kind" ); |
| 224 | } |
| 225 | |
| 226 | /// Return the intrinsic ID associated with stmatrix for the given paramters. |
| 227 | static llvm::Intrinsic::ID |
| 228 | getStMatrixIntrinsicId(NVVM::MMALayout layout, int32_t num, |
| 229 | NVVM::LdStMatrixShapeAttr shape, |
| 230 | NVVM::LdStMatrixEltType eltType) { |
| 231 | if (shape.getM() == 8 && shape.getN() == 8) { |
| 232 | switch (num) { |
| 233 | case 1: |
| 234 | return (layout == NVVM::MMALayout::row) |
| 235 | ? llvm::Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x1_b16 |
| 236 | : llvm::Intrinsic:: |
| 237 | nvvm_stmatrix_sync_aligned_m8n8_x1_trans_b16; |
| 238 | case 2: |
| 239 | return (layout == NVVM::MMALayout::row) |
| 240 | ? llvm::Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x2_b16 |
| 241 | : llvm::Intrinsic:: |
| 242 | nvvm_stmatrix_sync_aligned_m8n8_x2_trans_b16; |
| 243 | case 4: |
| 244 | return (layout == NVVM::MMALayout::row) |
| 245 | ? llvm::Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x4_b16 |
| 246 | : llvm::Intrinsic:: |
| 247 | nvvm_stmatrix_sync_aligned_m8n8_x4_trans_b16; |
| 248 | } |
| 249 | } else if (shape.getM() == 16 && shape.getN() == 8) { |
| 250 | switch (num) { |
| 251 | case 1: |
| 252 | return llvm::Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x1_trans_b8; |
| 253 | case 2: |
| 254 | return llvm::Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x2_trans_b8; |
| 255 | case 4: |
| 256 | return llvm::Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x4_trans_b8; |
| 257 | } |
| 258 | } |
| 259 | llvm_unreachable("unknown stmatrix kind" ); |
| 260 | } |
| 261 | |
| 262 | static unsigned getUnidirectionalFenceProxyID(NVVM::ProxyKind fromProxy, |
| 263 | NVVM::ProxyKind toProxy, |
| 264 | NVVM::MemScopeKind scope, |
| 265 | bool isRelease) { |
| 266 | if (fromProxy == NVVM::ProxyKind::GENERIC && |
| 267 | toProxy == NVVM::ProxyKind::TENSORMAP) { |
| 268 | switch (scope) { |
| 269 | case NVVM::MemScopeKind::CTA: { |
| 270 | if (isRelease) |
| 271 | return llvm::Intrinsic::nvvm_fence_proxy_tensormap_generic_release_cta; |
| 272 | return llvm::Intrinsic::nvvm_fence_proxy_tensormap_generic_acquire_cta; |
| 273 | } |
| 274 | case NVVM::MemScopeKind::CLUSTER: { |
| 275 | if (isRelease) |
| 276 | return llvm::Intrinsic:: |
| 277 | nvvm_fence_proxy_tensormap_generic_release_cluster; |
| 278 | return llvm::Intrinsic:: |
| 279 | nvvm_fence_proxy_tensormap_generic_acquire_cluster; |
| 280 | } |
| 281 | case NVVM::MemScopeKind::GPU: { |
| 282 | if (isRelease) |
| 283 | return llvm::Intrinsic::nvvm_fence_proxy_tensormap_generic_release_gpu; |
| 284 | return llvm::Intrinsic::nvvm_fence_proxy_tensormap_generic_acquire_gpu; |
| 285 | } |
| 286 | case NVVM::MemScopeKind::SYS: { |
| 287 | if (isRelease) |
| 288 | return llvm::Intrinsic::nvvm_fence_proxy_tensormap_generic_release_sys; |
| 289 | return llvm::Intrinsic::nvvm_fence_proxy_tensormap_generic_acquire_sys; |
| 290 | } |
| 291 | } |
| 292 | llvm_unreachable("Unknown scope for uni-directional fence.proxy operation" ); |
| 293 | } |
| 294 | llvm_unreachable("Unsupported proxy kinds" ); |
| 295 | } |
| 296 | |
| 297 | static unsigned getMembarIntrinsicID(NVVM::MemScopeKind scope) { |
| 298 | switch (scope) { |
| 299 | case NVVM::MemScopeKind::CTA: |
| 300 | return llvm::Intrinsic::nvvm_membar_cta; |
| 301 | case NVVM::MemScopeKind::CLUSTER: |
| 302 | return llvm::Intrinsic::nvvm_fence_sc_cluster; |
| 303 | case NVVM::MemScopeKind::GPU: |
| 304 | return llvm::Intrinsic::nvvm_membar_gl; |
| 305 | case NVVM::MemScopeKind::SYS: |
| 306 | return llvm::Intrinsic::nvvm_membar_sys; |
| 307 | } |
| 308 | llvm_unreachable("Unknown scope for memory barrier" ); |
| 309 | } |
| 310 | |
| 311 | #define TCGEN05LD(SHAPE, NUM) llvm::Intrinsic::nvvm_tcgen05_ld_##SHAPE##_##NUM |
| 312 | |
| 313 | static llvm::Intrinsic::ID |
| 314 | getTcgen05LdIntrinsicID(mlir::NVVM::Tcgen05LdStShape shape, uint32_t num) { |
| 315 | llvm::Intrinsic::ID Shape16x64b[] = { |
| 316 | TCGEN05LD(16x64b, x1), TCGEN05LD(16x64b, x2), TCGEN05LD(16x64b, x4), |
| 317 | TCGEN05LD(16x64b, x8), TCGEN05LD(16x64b, x16), TCGEN05LD(16x64b, x32), |
| 318 | TCGEN05LD(16x64b, x64), TCGEN05LD(16x64b, x128), |
| 319 | }; |
| 320 | |
| 321 | llvm::Intrinsic::ID Shape16x128b[] = { |
| 322 | TCGEN05LD(16x128b, x1), TCGEN05LD(16x128b, x2), TCGEN05LD(16x128b, x4), |
| 323 | TCGEN05LD(16x128b, x8), TCGEN05LD(16x128b, x16), TCGEN05LD(16x128b, x32), |
| 324 | TCGEN05LD(16x128b, x64), |
| 325 | }; |
| 326 | |
| 327 | llvm::Intrinsic::ID Shape16x256b[] = { |
| 328 | TCGEN05LD(16x256b, x1), TCGEN05LD(16x256b, x2), TCGEN05LD(16x256b, x4), |
| 329 | TCGEN05LD(16x256b, x8), TCGEN05LD(16x256b, x16), TCGEN05LD(16x256b, x32), |
| 330 | }; |
| 331 | |
| 332 | llvm::Intrinsic::ID Shape16x32bx2[] = { |
| 333 | TCGEN05LD(16x32bx2, x1), TCGEN05LD(16x32bx2, x2), |
| 334 | TCGEN05LD(16x32bx2, x4), TCGEN05LD(16x32bx2, x8), |
| 335 | TCGEN05LD(16x32bx2, x16), TCGEN05LD(16x32bx2, x32), |
| 336 | TCGEN05LD(16x32bx2, x64), TCGEN05LD(16x32bx2, x128), |
| 337 | }; |
| 338 | |
| 339 | llvm::Intrinsic::ID Shape32x32b[] = { |
| 340 | TCGEN05LD(32x32b, x1), TCGEN05LD(32x32b, x2), TCGEN05LD(32x32b, x4), |
| 341 | TCGEN05LD(32x32b, x8), TCGEN05LD(32x32b, x16), TCGEN05LD(32x32b, x32), |
| 342 | TCGEN05LD(32x32b, x64), TCGEN05LD(32x32b, x128), |
| 343 | }; |
| 344 | |
| 345 | // `num` contains the length of vector and log2 of `num` returns the index |
| 346 | // into the shape array |
| 347 | unsigned Idx = std::log2(x: num); |
| 348 | |
| 349 | switch (shape) { |
| 350 | case NVVM::Tcgen05LdStShape::SHAPE_16X64B: |
| 351 | return Shape16x64b[Idx]; |
| 352 | case NVVM::Tcgen05LdStShape::SHAPE_16X128B: |
| 353 | return Shape16x128b[Idx - 1]; |
| 354 | case NVVM::Tcgen05LdStShape::SHAPE_16X256B: |
| 355 | return Shape16x256b[Idx - 2]; |
| 356 | case NVVM::Tcgen05LdStShape::SHAPE_32X32B: |
| 357 | return Shape32x32b[Idx]; |
| 358 | case NVVM::Tcgen05LdStShape::SHAPE_16X32BX2: |
| 359 | return Shape16x32bx2[Idx]; |
| 360 | } |
| 361 | llvm_unreachable("unhandled tcgen05.ld lowering" ); |
| 362 | } |
| 363 | |
| 364 | #define TCGEN05ST(SHAPE, NUM) llvm::Intrinsic::nvvm_tcgen05_st_##SHAPE##_##NUM |
| 365 | |
| 366 | static llvm::Intrinsic::ID |
| 367 | getTcgen05StIntrinsicID(mlir::NVVM::Tcgen05LdStShape shape, uint32_t num) { |
| 368 | llvm::Intrinsic::ID Shape16x64b[] = { |
| 369 | TCGEN05ST(16x64b, x1), TCGEN05ST(16x64b, x2), TCGEN05ST(16x64b, x4), |
| 370 | TCGEN05ST(16x64b, x8), TCGEN05ST(16x64b, x16), TCGEN05ST(16x64b, x32), |
| 371 | TCGEN05ST(16x64b, x64), TCGEN05ST(16x64b, x128), |
| 372 | }; |
| 373 | |
| 374 | llvm::Intrinsic::ID Shape16x128b[] = { |
| 375 | TCGEN05ST(16x128b, x1), TCGEN05ST(16x128b, x2), TCGEN05ST(16x128b, x4), |
| 376 | TCGEN05ST(16x128b, x8), TCGEN05ST(16x128b, x16), TCGEN05ST(16x128b, x32), |
| 377 | TCGEN05ST(16x128b, x64), |
| 378 | }; |
| 379 | |
| 380 | llvm::Intrinsic::ID Shape16x256b[] = { |
| 381 | TCGEN05ST(16x256b, x1), TCGEN05ST(16x256b, x2), TCGEN05ST(16x256b, x4), |
| 382 | TCGEN05ST(16x256b, x8), TCGEN05ST(16x256b, x16), TCGEN05ST(16x256b, x32), |
| 383 | }; |
| 384 | |
| 385 | llvm::Intrinsic::ID Shape16x32bx2[] = { |
| 386 | TCGEN05ST(16x32bx2, x1), TCGEN05ST(16x32bx2, x2), |
| 387 | TCGEN05ST(16x32bx2, x4), TCGEN05ST(16x32bx2, x8), |
| 388 | TCGEN05ST(16x32bx2, x16), TCGEN05ST(16x32bx2, x32), |
| 389 | TCGEN05ST(16x32bx2, x64), TCGEN05ST(16x32bx2, x128), |
| 390 | }; |
| 391 | |
| 392 | llvm::Intrinsic::ID Shape32x32b[] = { |
| 393 | TCGEN05ST(32x32b, x1), TCGEN05ST(32x32b, x2), TCGEN05ST(32x32b, x4), |
| 394 | TCGEN05ST(32x32b, x8), TCGEN05ST(32x32b, x16), TCGEN05ST(32x32b, x32), |
| 395 | TCGEN05ST(32x32b, x64), TCGEN05ST(32x32b, x128), |
| 396 | }; |
| 397 | |
| 398 | // `num` contains the length of vector and log2 of `num` returns the index |
| 399 | // into the shape array |
| 400 | unsigned Idx = std::log2(x: num); |
| 401 | |
| 402 | switch (shape) { |
| 403 | case NVVM::Tcgen05LdStShape::SHAPE_16X64B: |
| 404 | return Shape16x64b[Idx]; |
| 405 | case NVVM::Tcgen05LdStShape::SHAPE_16X128B: |
| 406 | return Shape16x128b[Idx - 1]; |
| 407 | case NVVM::Tcgen05LdStShape::SHAPE_16X256B: |
| 408 | return Shape16x256b[Idx - 2]; |
| 409 | case NVVM::Tcgen05LdStShape::SHAPE_32X32B: |
| 410 | return Shape32x32b[Idx]; |
| 411 | case NVVM::Tcgen05LdStShape::SHAPE_16X32BX2: |
| 412 | return Shape16x32bx2[Idx]; |
| 413 | } |
| 414 | llvm_unreachable("unhandled tcgen05.st lowering" ); |
| 415 | } |
| 416 | |
| 417 | static llvm::Intrinsic::ID getFenceSyncRestrictID(NVVM::MemOrderKind order) { |
| 418 | return order == NVVM::MemOrderKind::ACQUIRE |
| 419 | ? llvm::Intrinsic:: |
| 420 | nvvm_fence_acquire_sync_restrict_space_cluster_scope_cluster |
| 421 | : llvm::Intrinsic:: |
| 422 | nvvm_fence_release_sync_restrict_space_cta_scope_cluster; |
| 423 | } |
| 424 | |
| 425 | static llvm::Intrinsic::ID |
| 426 | getFenceProxyID(NVVM::ProxyKind kind, std::optional<NVVM::SharedSpace> space) { |
| 427 | switch (kind) { |
| 428 | case NVVM::ProxyKind::alias: |
| 429 | return llvm::Intrinsic::nvvm_fence_proxy_alias; |
| 430 | case NVVM::ProxyKind::async: |
| 431 | return llvm::Intrinsic::nvvm_fence_proxy_async; |
| 432 | case NVVM::ProxyKind::async_global: |
| 433 | return llvm::Intrinsic::nvvm_fence_proxy_async_global; |
| 434 | case NVVM::ProxyKind::async_shared: |
| 435 | return *space == NVVM::SharedSpace::shared_cta |
| 436 | ? llvm::Intrinsic::nvvm_fence_proxy_async_shared_cta |
| 437 | : llvm::Intrinsic::nvvm_fence_proxy_async_shared_cluster; |
| 438 | default: |
| 439 | llvm_unreachable("unsupported proxy kind" ); |
| 440 | } |
| 441 | } |
| 442 | |
| 443 | static llvm::Intrinsic::ID |
| 444 | getFenceProxySyncRestrictID(NVVM::MemOrderKind order) { |
| 445 | return order == NVVM::MemOrderKind::ACQUIRE |
| 446 | ? llvm::Intrinsic:: |
| 447 | nvvm_fence_proxy_async_generic_acquire_sync_restrict_space_cluster_scope_cluster |
| 448 | : llvm::Intrinsic:: |
| 449 | nvvm_fence_proxy_async_generic_release_sync_restrict_space_cta_scope_cluster; |
| 450 | } |
| 451 | |
| 452 | static llvm::RoundingMode |
| 453 | getLLVMRoundingModeForFPArith(NVVM::FPRoundingMode rndMode) { |
| 454 | switch (rndMode) { |
| 455 | case NVVM::FPRoundingMode::RN: |
| 456 | return llvm::RoundingMode::NearestTiesToEven; |
| 457 | case NVVM::FPRoundingMode::RM: |
| 458 | return llvm::RoundingMode::TowardNegative; |
| 459 | case NVVM::FPRoundingMode::RP: |
| 460 | return llvm::RoundingMode::TowardPositive; |
| 461 | case NVVM::FPRoundingMode::RZ: |
| 462 | return llvm::RoundingMode::TowardZero; |
| 463 | default: |
| 464 | // default rounding mode is RN |
| 465 | assert(rndMode == NVVM::FPRoundingMode::NONE && |
| 466 | "unsupported rounding mode for nvvm fp arithmetic" ); |
| 467 | return llvm::RoundingMode::NearestTiesToEven; |
| 468 | } |
| 469 | } |
| 470 | |
| 471 | // Calls an LLVM intrinsic on the given operands. For f32/f64 vector types, |
| 472 | // the intrinsic is called per-element and the results are packed back into a |
| 473 | // vector. If retType is non-null, it is forwarded as the return-type |
| 474 | // overload to `createIntrinsicCall`. |
| 475 | static llvm::Value * |
| 476 | createScalarizedIntrinsicCall(llvm::IRBuilderBase &builder, |
| 477 | llvm::Intrinsic::ID IID, llvm::Type *opTypeLLVM, |
| 478 | ArrayRef<llvm::Value *> operands, |
| 479 | llvm::Type *retType) { |
| 480 | if (opTypeLLVM->isVectorTy() && (opTypeLLVM->getScalarType()->isFloatTy() || |
| 481 | opTypeLLVM->getScalarType()->isDoubleTy())) { |
| 482 | llvm::Value *result = llvm::PoisonValue::get( |
| 483 | T: llvm::FixedVectorType::get(ElementType: opTypeLLVM->getScalarType(), NumElts: 2)); |
| 484 | for (int64_t i = 0; i < 2; ++i) { |
| 485 | llvm::SmallVector<llvm::Value *> scalarArgs; |
| 486 | for (llvm::Value *op : operands) |
| 487 | scalarArgs.push_back( |
| 488 | Elt: op->getType()->isVectorTy() |
| 489 | ? builder.CreateExtractElement(Vec: op, Idx: builder.getInt32(C: i)) |
| 490 | : op); |
| 491 | llvm::Value *res = createIntrinsicCall(builder, intrinsic: IID, retTy: retType, args: scalarArgs); |
| 492 | result = builder.CreateInsertElement(Vec: result, NewElt: res, Idx: builder.getInt32(C: i)); |
| 493 | } |
| 494 | return result; |
| 495 | } |
| 496 | |
| 497 | return createIntrinsicCall(builder, intrinsic: IID, retTy: retType, args: operands); |
| 498 | } |
| 499 | |
| 500 | // Lowers a binary FP arithmetic op to the intrinsic selected from `ids`, which |
| 501 | // is indexed by [ftz][sat]. |
| 502 | static void lowerFloatBinaryOpToLLVMIR(const llvm::Intrinsic::ID (&ids)[2][2], |
| 503 | llvm::Value *argLHS, llvm::Value *argRHS, |
| 504 | Value res, NVVM::FPRoundingMode rndMode, |
| 505 | NVVM::SaturationMode satMode, bool isFTZ, |
| 506 | LLVM::ModuleTranslation &mt, |
| 507 | llvm::IRBuilderBase &builder) { |
| 508 | llvm::Type *opTypeLLVM = argLHS->getType(); |
| 509 | bool isSat = satMode != NVVM::SaturationMode::NONE; |
| 510 | |
| 511 | llvm::Intrinsic::ID id = ids[isFTZ][isSat]; |
| 512 | llvm::Value *rnd = builder.getInt32( |
| 513 | C: static_cast<int>(getLLVMRoundingModeForFPArith(rndMode))); |
| 514 | |
| 515 | // For f64 vectors, and f32 vectors with saturation, we need to scalarize |
| 516 | // the intrinsic call. |
| 517 | llvm::Type *scalarTypeLLVM = opTypeLLVM->getScalarType(); |
| 518 | if (opTypeLLVM->isVectorTy() && (scalarTypeLLVM->isDoubleTy() || |
| 519 | (isSat && scalarTypeLLVM->isFloatTy()))) { |
| 520 | mt.mapValue(mlir: res, llvm: createScalarizedIntrinsicCall(builder, IID: id, opTypeLLVM, |
| 521 | operands: {argLHS, argRHS, rnd}, |
| 522 | retType: scalarTypeLLVM)); |
| 523 | return; |
| 524 | } |
| 525 | |
| 526 | mt.mapValue( |
| 527 | mlir: res, llvm: createIntrinsicCall(builder, intrinsic: id, retTy: opTypeLLVM, args: {argLHS, argRHS, rnd})); |
| 528 | } |
| 529 | |
| 530 | void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS, |
| 531 | Value res, NVVM::FPRoundingMode rndMode, |
| 532 | NVVM::SaturationMode satMode, bool isFTZ, |
| 533 | LLVM::ModuleTranslation &mt, |
| 534 | llvm::IRBuilderBase &builder) { |
| 535 | static constexpr llvm::Intrinsic::ID addIDs[2][2] = { |
| 536 | {llvm::Intrinsic::nvvm_fadd, llvm::Intrinsic::nvvm_fadd_sat}, |
| 537 | {llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_ftz_sat}}; |
| 538 | lowerFloatBinaryOpToLLVMIR(ids: addIDs, argLHS, argRHS, res, rndMode, satMode, |
| 539 | isFTZ, mt, builder); |
| 540 | } |
| 541 | |
| 542 | void NVVM::MulFOp::lowerMulFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS, |
| 543 | Value res, NVVM::FPRoundingMode rndMode, |
| 544 | NVVM::SaturationMode satMode, bool isFTZ, |
| 545 | LLVM::ModuleTranslation &mt, |
| 546 | llvm::IRBuilderBase &builder) { |
| 547 | static constexpr llvm::Intrinsic::ID mulIDs[2][2] = { |
| 548 | {llvm::Intrinsic::nvvm_fmul, llvm::Intrinsic::nvvm_fmul_sat}, |
| 549 | {llvm::Intrinsic::nvvm_fmul_ftz, llvm::Intrinsic::nvvm_fmul_ftz_sat}}; |
| 550 | lowerFloatBinaryOpToLLVMIR(ids: mulIDs, argLHS, argRHS, res, rndMode, satMode, |
| 551 | isFTZ, mt, builder); |
| 552 | } |
| 553 | |
| 554 | void NVVM::FmaOp::lowerFmaToLLVMIR(Operation &op, LLVM::ModuleTranslation &mt, |
| 555 | llvm::IRBuilderBase &builder) { |
| 556 | auto thisOp = cast<NVVM::FmaOp>(Val&: op); |
| 557 | mlir::NVVM::FPRoundingMode rndMode = thisOp.getRnd(); |
| 558 | bool isFTZ = thisOp.getFtz(); |
| 559 | bool isRelu = thisOp.getRelu(); |
| 560 | bool isSat = thisOp.getSat() == NVVM::SaturationMode::SAT; |
| 561 | bool isOOB = thisOp.getOob(); |
| 562 | |
| 563 | mlir::Type opType = thisOp.getRes().getType(); |
| 564 | llvm::Type *opTypeLLVM = mt.convertType(type: opType); |
| 565 | |
| 566 | llvm::Value *argA = mt.lookupValue(value: thisOp.getA()); |
| 567 | llvm::Value *argB = mt.lookupValue(value: thisOp.getB()); |
| 568 | llvm::Value *argC = mt.lookupValue(value: thisOp.getC()); |
| 569 | |
| 570 | // The oob variants only support RN and take no rounding-mode operand. |
| 571 | if (isOOB) { |
| 572 | llvm::Intrinsic::ID id = isRelu ? llvm::Intrinsic::nvvm_fma_rn_oob_relu |
| 573 | : llvm::Intrinsic::nvvm_fma_rn_oob; |
| 574 | mt.mapValue(mlir: thisOp.getRes(), llvm: createIntrinsicCall(builder, intrinsic: id, retTy: opTypeLLVM, |
| 575 | args: {argA, argB, argC})); |
| 576 | return; |
| 577 | } |
| 578 | |
| 579 | static constexpr llvm::Intrinsic::ID fmaIDs[2][3] = { |
| 580 | {llvm::Intrinsic::nvvm_fma, llvm::Intrinsic::nvvm_fma_sat, |
| 581 | llvm::Intrinsic::nvvm_fma_relu}, |
| 582 | {llvm::Intrinsic::nvvm_fma_ftz, llvm::Intrinsic::nvvm_fma_ftz_sat, |
| 583 | llvm::Intrinsic::nvvm_fma_ftz_relu}}; |
| 584 | |
| 585 | llvm::Intrinsic::ID id = fmaIDs[isFTZ][isSat ? 1 : (isRelu ? 2 : 0)]; |
| 586 | llvm::Value *rnd = builder.getInt32( |
| 587 | C: static_cast<int>(getLLVMRoundingModeForFPArith(rndMode))); |
| 588 | |
| 589 | // For f64 vector fma, and f32 vector fma with saturation, we need to |
| 590 | // scalarize the intrinsic call. |
| 591 | llvm::Type *scalarTypeLLVM = opTypeLLVM->getScalarType(); |
| 592 | if (opTypeLLVM->isVectorTy() && (scalarTypeLLVM->isDoubleTy() || |
| 593 | (isSat && scalarTypeLLVM->isFloatTy()))) { |
| 594 | mt.mapValue(mlir: thisOp.getRes(), llvm: createScalarizedIntrinsicCall( |
| 595 | builder, IID: id, opTypeLLVM, |
| 596 | operands: {argA, argB, argC, rnd}, retType: scalarTypeLLVM)); |
| 597 | return; |
| 598 | } |
| 599 | |
| 600 | mt.mapValue(mlir: thisOp.getRes(), llvm: createIntrinsicCall(builder, intrinsic: id, retTy: opTypeLLVM, |
| 601 | args: {argA, argB, argC, rnd})); |
| 602 | } |
| 603 | |
| 604 | // Returns the LLVM type the sp{de}compress intrinsics use for metadata: i32 for |
| 605 | // a single register (including vector<1xi32>) and a vector of i32 otherwise. |
| 606 | static llvm::Type *getSPMetadataType(Type type, llvm::IRBuilderBase &builder) { |
| 607 | auto vecTy = dyn_cast<VectorType>(Val&: type); |
| 608 | if (!vecTy || vecTy.getNumElements() == 1) |
| 609 | return builder.getInt32Ty(); |
| 610 | return llvm::FixedVectorType::get(ElementType: builder.getInt32Ty(), |
| 611 | NumElts: vecTy.getNumElements()); |
| 612 | } |
| 613 | |
| 614 | void NVVM::SparseCompressOp::lowerToLLVMIR(Operation &op, |
| 615 | LLVM::ModuleTranslation &mt, |
| 616 | llvm::IRBuilderBase &builder) { |
| 617 | auto thisOp = cast<NVVM::SparseCompressOp>(Val&: op); |
| 618 | Value metadata = thisOp.getMetadata(); |
| 619 | Value compressedData = thisOp.getCompressedData(); |
| 620 | llvm::Type *resultTy = |
| 621 | llvm::StructType::get(elt1: getSPMetadataType(type: metadata.getType(), builder), |
| 622 | elts: mt.convertType(type: compressedData.getType())); |
| 623 | |
| 624 | llvm::Value *result = createIntrinsicCall( |
| 625 | builder, intrinsic: llvm::Intrinsic::nvvm_spcompress, retTy: resultTy, |
| 626 | args: {mt.lookupValue(value: thisOp.getData()), mt.lookupValue(value: thisOp.getSpDesc()), |
| 627 | builder.getInt32(C: thisOp.getIdxSize()), |
| 628 | // spcompress only supports the 2:4 sparsity factor. |
| 629 | builder.getInt32(C: 4)}); |
| 630 | |
| 631 | mt.mapValue(value: metadata) = |
| 632 | builder.CreateBitCast(V: builder.CreateExtractValue(Agg: result, Idxs: 0), |
| 633 | DestTy: mt.convertType(type: metadata.getType())); |
| 634 | mt.mapValue(value: compressedData) = builder.CreateExtractValue(Agg: result, Idxs: 1); |
| 635 | } |
| 636 | |
| 637 | void NVVM::SparseDecompressOp::lowerToLLVMIR(Operation &op, |
| 638 | LLVM::ModuleTranslation &mt, |
| 639 | llvm::IRBuilderBase &builder) { |
| 640 | auto thisOp = cast<NVVM::SparseDecompressOp>(Val&: op); |
| 641 | Value metadata = thisOp.getMetadata(); |
| 642 | mt.mapValue(value: thisOp.getData()) = createIntrinsicCall( |
| 643 | builder, intrinsic: llvm::Intrinsic::nvvm_spdecompress, |
| 644 | retTy: mt.convertType(type: thisOp.getData().getType()), |
| 645 | args: {builder.CreateBitCast(V: mt.lookupValue(value: metadata), |
| 646 | DestTy: getSPMetadataType(type: metadata.getType(), builder)), |
| 647 | mt.lookupValue(value: thisOp.getCompressedData()), |
| 648 | builder.getInt32(C: thisOp.getIdxSize()), |
| 649 | builder.getInt32(C: thisOp.getNumTgt())}); |
| 650 | } |
| 651 | |
| 652 | namespace { |
| 653 | /// Implementation of the dialect interface that converts operations belonging |
| 654 | /// to the NVVM dialect to LLVM IR. |
| 655 | class NVVMDialectLLVMIRTranslationInterface |
| 656 | : public LLVMTranslationDialectInterface { |
| 657 | public: |
| 658 | using LLVMTranslationDialectInterface::LLVMTranslationDialectInterface; |
| 659 | |
| 660 | /// Translates the given operation to LLVM IR using the provided IR builder |
| 661 | /// and saving the state in `moduleTranslation`. |
| 662 | LogicalResult |
| 663 | convertOperation(Operation *op, llvm::IRBuilderBase &builder, |
| 664 | LLVM::ModuleTranslation &moduleTranslation) const final { |
| 665 | // All NVVM ops are instruction-level and require an active insertion point. |
| 666 | // A null insert block means the op is misplaced (e.g., at module scope), |
| 667 | // which would otherwise cause a null dereference in createIntrinsicCall. |
| 668 | if (!builder.GetInsertBlock()) |
| 669 | return op->emitOpError( |
| 670 | message: "cannot be translated to LLVM IR without an active insertion " |
| 671 | "point; make sure the op is inside a function" ); |
| 672 | Operation &opInst = *op; |
| 673 | #include "mlir/Dialect/LLVMIR/NVVMConversions.inc" |
| 674 | |
| 675 | return failure(); |
| 676 | } |
| 677 | |
| 678 | /// Attaches module-level metadata for functions marked as kernels |
| 679 | /// and managed annotations for global variables. |
| 680 | LogicalResult |
| 681 | amendOperation(Operation *op, ArrayRef<llvm::Instruction *> instructions, |
| 682 | NamedAttribute attribute, |
| 683 | LLVM::ModuleTranslation &moduleTranslation) const final { |
| 684 | if (auto globalOp = dyn_cast<LLVM::GlobalOp>(Val: op)) { |
| 685 | if (attribute.getName() == NVVM::NVVMDialect::getManagedAttrName()) { |
| 686 | auto *gv = cast<llvm::GlobalVariable>( |
| 687 | Val: moduleTranslation.lookupGlobal(op: globalOp)); |
| 688 | llvm::Module *m = gv->getParent(); |
| 689 | llvm::LLVMContext &ctx = m->getContext(); |
| 690 | llvm::NamedMDNode *md = m->getOrInsertNamedMetadata(Name: "nvvm.annotations" ); |
| 691 | md->addOperand(M: llvm::MDNode::get( |
| 692 | Context&: ctx, MDs: {llvm::ConstantAsMetadata::get(C: gv), |
| 693 | llvm::MDString::get(Context&: ctx, Str: "managed" ), |
| 694 | llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get( |
| 695 | Ty: llvm::Type::getInt32Ty(C&: ctx), V: 1))})); |
| 696 | } |
| 697 | return success(); |
| 698 | } |
| 699 | |
| 700 | auto func = dyn_cast<LLVM::LLVMFuncOp>(Val: op); |
| 701 | if (!func) |
| 702 | return failure(); |
| 703 | llvm::Function *llvmFunc = moduleTranslation.lookupFunction(name: func.getName()); |
| 704 | |
| 705 | if (attribute.getName() == NVVM::NVVMDialect::getMaxntidAttrName()) { |
| 706 | if (!isa<DenseI32ArrayAttr>(Val: attribute.getValue())) |
| 707 | return failure(); |
| 708 | auto values = cast<DenseI32ArrayAttr>(Val: attribute.getValue()); |
| 709 | const std::string attr = llvm::formatv( |
| 710 | Fmt: "{0:$[,]}" , Vals: llvm::make_range(x: values.asArrayRef().begin(), |
| 711 | y: values.asArrayRef().end())); |
| 712 | llvmFunc->addFnAttr(Kind: llvm::NVVMAttr::MaxNTID, Val: attr); |
| 713 | } else if (attribute.getName() == NVVM::NVVMDialect::getReqntidAttrName()) { |
| 714 | if (!isa<DenseI32ArrayAttr>(Val: attribute.getValue())) |
| 715 | return failure(); |
| 716 | auto values = cast<DenseI32ArrayAttr>(Val: attribute.getValue()); |
| 717 | const std::string attr = llvm::formatv( |
| 718 | Fmt: "{0:$[,]}" , Vals: llvm::make_range(x: values.asArrayRef().begin(), |
| 719 | y: values.asArrayRef().end())); |
| 720 | llvmFunc->addFnAttr(Kind: llvm::NVVMAttr::ReqNTID, Val: attr); |
| 721 | } else if (attribute.getName() == |
| 722 | NVVM::NVVMDialect::getClusterDimAttrName()) { |
| 723 | if (!isa<DenseI32ArrayAttr>(Val: attribute.getValue())) |
| 724 | return failure(); |
| 725 | auto values = cast<DenseI32ArrayAttr>(Val: attribute.getValue()); |
| 726 | const std::string attr = llvm::formatv( |
| 727 | Fmt: "{0:$[,]}" , Vals: llvm::make_range(x: values.asArrayRef().begin(), |
| 728 | y: values.asArrayRef().end())); |
| 729 | llvmFunc->addFnAttr(Kind: llvm::NVVMAttr::ClusterDim, Val: attr); |
| 730 | } else if (attribute.getName() == |
| 731 | NVVM::NVVMDialect::getClusterMaxBlocksAttrName()) { |
| 732 | auto value = dyn_cast<IntegerAttr>(Val: attribute.getValue()); |
| 733 | llvmFunc->addFnAttr(Kind: llvm::NVVMAttr::MaxClusterRank, |
| 734 | Val: llvm::utostr(X: value.getInt())); |
| 735 | } else if (attribute.getName() == |
| 736 | NVVM::NVVMDialect::getMinctasmAttrName()) { |
| 737 | auto value = dyn_cast<IntegerAttr>(Val: attribute.getValue()); |
| 738 | llvmFunc->addFnAttr(Kind: llvm::NVVMAttr::MinCTASm, |
| 739 | Val: llvm::utostr(X: value.getInt())); |
| 740 | } else if (attribute.getName() == NVVM::NVVMDialect::getMaxnregAttrName()) { |
| 741 | auto value = dyn_cast<IntegerAttr>(Val: attribute.getValue()); |
| 742 | llvmFunc->addFnAttr(Kind: llvm::NVVMAttr::MaxNReg, |
| 743 | Val: llvm::utostr(X: value.getInt())); |
| 744 | } else if (attribute.getName() == |
| 745 | NVVM::NVVMDialect::getKernelFuncAttrName()) { |
| 746 | llvmFunc->setCallingConv(llvm::CallingConv::PTX_Kernel); |
| 747 | } else if (attribute.getName() == |
| 748 | NVVM::NVVMDialect::getBlocksAreClustersAttrName()) { |
| 749 | llvmFunc->addFnAttr(Kind: llvm::NVVMAttr::BlocksAreClusters); |
| 750 | } |
| 751 | |
| 752 | return success(); |
| 753 | } |
| 754 | |
| 755 | LogicalResult |
| 756 | convertParameterAttr(LLVMFuncOp funcOp, int argIdx, NamedAttribute attribute, |
| 757 | LLVM::ModuleTranslation &moduleTranslation) const final { |
| 758 | |
| 759 | llvm::LLVMContext &llvmContext = moduleTranslation.getLLVMContext(); |
| 760 | llvm::Function *llvmFunc = |
| 761 | moduleTranslation.lookupFunction(name: funcOp.getName()); |
| 762 | |
| 763 | if (attribute.getName() == NVVM::NVVMDialect::getGridConstantAttrName()) { |
| 764 | llvmFunc->addParamAttr( |
| 765 | ArgNo: argIdx, |
| 766 | Attr: llvm::Attribute::get(Context&: llvmContext, Kind: llvm::NVVMAttr::GridConstant)); |
| 767 | } |
| 768 | return success(); |
| 769 | } |
| 770 | }; |
| 771 | } // namespace |
| 772 | |
| 773 | void mlir::registerNVVMDialectTranslation(DialectRegistry ®istry) { |
| 774 | registry.insert<NVVM::NVVMDialect>(); |
| 775 | registry.addExtension(extensionFn: +[](MLIRContext *ctx, NVVM::NVVMDialect *dialect) { |
| 776 | dialect->addInterfaces<NVVMDialectLLVMIRTranslationInterface>(); |
| 777 | }); |
| 778 | } |
| 779 | |
| 780 | void mlir::registerNVVMDialectTranslation(MLIRContext &context) { |
| 781 | DialectRegistry registry; |
| 782 | registerNVVMDialectTranslation(registry); |
| 783 | context.appendDialectRegistry(registry); |
| 784 | } |
| 785 | |