| 1 | //===- LLVMIRToNVVMTranslation.cpp - Translate LLVM IR to NVVM dialect ----===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements a translation between LLVM IR and the MLIR NVVM dialect. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Target/LLVMIR/Dialect/NVVM/LLVMIRToNVVMTranslation.h" |
| 14 | #include "mlir/Dialect/LLVMIR/NVVMDialect.h" |
| 15 | #include "mlir/Target/LLVMIR/ModuleImport.h" |
| 16 | #include "llvm/IR/ConstantRange.h" |
| 17 | |
| 18 | using namespace mlir; |
| 19 | using namespace mlir::NVVM; |
| 20 | |
| 21 | /// LLVM intrinsic IDs of the four `bar.sync` variants imported into |
| 22 | /// `nvvm.barrier`. |
| 23 | static constexpr llvm::Intrinsic::ID kBarrierSyncIntrinsics[] = { |
| 24 | llvm::Intrinsic::nvvm_barrier_cta_sync_all, |
| 25 | llvm::Intrinsic::nvvm_barrier_cta_sync_aligned_all, |
| 26 | llvm::Intrinsic::nvvm_barrier_cta_sync_count, |
| 27 | llvm::Intrinsic::nvvm_barrier_cta_sync_aligned_count, |
| 28 | }; |
| 29 | |
| 30 | /// Returns true if `id` is one of the `bar.sync` intrinsic IDs imported into |
| 31 | /// `nvvm.barrier`. |
| 32 | static bool isBarrierSyncIntrinsic(llvm::Intrinsic::ID id) { |
| 33 | return llvm::is_contained(Range: kBarrierSyncIntrinsics, Element: id); |
| 34 | } |
| 35 | |
| 36 | /// Imports one of the four `bar.sync` LLVM intrinsic variants into a single |
| 37 | /// `nvvm.barrier` op, deriving the `aligned` attribute and the optional |
| 38 | /// `numberOfThreads` operand from the specific intrinsic ID. |
| 39 | static LogicalResult |
| 40 | convertBarrierSyncIntrinsic(OpBuilder &odsBuilder, llvm::CallInst *inst, |
| 41 | LLVM::ModuleImport &moduleImport, |
| 42 | ArrayRef<llvm::Value *> llvmOperands, |
| 43 | ArrayRef<llvm::OperandBundleUse> llvmOpBundles) { |
| 44 | llvm::Intrinsic::ID id = inst->getIntrinsicID(); |
| 45 | bool aligned = (id == llvm::Intrinsic::nvvm_barrier_cta_sync_aligned_all || |
| 46 | id == llvm::Intrinsic::nvvm_barrier_cta_sync_aligned_count); |
| 47 | bool hasCount = (id == llvm::Intrinsic::nvvm_barrier_cta_sync_count || |
| 48 | id == llvm::Intrinsic::nvvm_barrier_cta_sync_aligned_count); |
| 49 | |
| 50 | SmallVector<Value> mlirOperands; |
| 51 | SmallVector<NamedAttribute> mlirAttrs; |
| 52 | if (failed(Result: moduleImport.convertIntrinsicArguments( |
| 53 | values: llvmOperands, opBundles: llvmOpBundles, /*requiresOpBundles=*/false, immArgPositions: {}, immArgAttrNames: {}, |
| 54 | valuesOut&: mlirOperands, attrsOut&: mlirAttrs))) |
| 55 | return failure(); |
| 56 | |
| 57 | auto op = NVVM::BarrierOp::create( |
| 58 | builder&: odsBuilder, location: moduleImport.translateLoc(loc: inst->getDebugLoc()), |
| 59 | barrierId: mlirOperands.front(), numberOfThreads: hasCount ? mlirOperands.back() : Value{}, |
| 60 | aligned: odsBuilder.getBoolAttr(value: aligned)); |
| 61 | moduleImport.mapNoResultOp(llvm: inst, mlir: op); |
| 62 | return success(); |
| 63 | } |
| 64 | |
| 65 | /// Returns true if the LLVM intrinsic `id` has a corresponding MLIR NVVM op. |
| 66 | static bool isConvertibleIntrinsic(llvm::Intrinsic::ID id) { |
| 67 | static const DenseSet<unsigned> convertibleIntrinsics = { |
| 68 | #include "mlir/Dialect/LLVMIR/NVVMConvertibleLLVMIRIntrinsics.inc" |
| 69 | }; |
| 70 | return convertibleIntrinsics.contains(V: id) || isBarrierSyncIntrinsic(id); |
| 71 | } |
| 72 | |
| 73 | /// Returns the LLVM intrinsic IDs that have a corresponding MLIR NVVM op. |
| 74 | static ArrayRef<unsigned> getSupportedIntrinsicsImpl() { |
| 75 | static const SmallVector<unsigned> convertibleIntrinsics = []() { |
| 76 | SmallVector<unsigned> ids = { |
| 77 | #include "mlir/Dialect/LLVMIR/NVVMConvertibleLLVMIRIntrinsics.inc" |
| 78 | }; |
| 79 | ids.append(in_start: std::begin(arr: kBarrierSyncIntrinsics), |
| 80 | in_end: std::end(arr: kBarrierSyncIntrinsics)); |
| 81 | return ids; |
| 82 | }(); |
| 83 | return convertibleIntrinsics; |
| 84 | } |
| 85 | |
| 86 | /// Converts the LLVM intrinsic to an MLIR NVVM dialect operation if a |
| 87 | /// conversion exits. Returns failure otherwise. |
| 88 | static LogicalResult convertIntrinsicImpl(OpBuilder &odsBuilder, |
| 89 | llvm::CallInst *inst, |
| 90 | LLVM::ModuleImport &moduleImport) { |
| 91 | llvm::Intrinsic::ID intrinsicID = inst->getIntrinsicID(); |
| 92 | |
| 93 | // Check if the intrinsic is convertible to an MLIR dialect counterpart and |
| 94 | // copy the arguments to an an LLVM operands array reference for conversion. |
| 95 | if (isConvertibleIntrinsic(id: intrinsicID)) { |
| 96 | SmallVector<llvm::Value *> args(inst->args()); |
| 97 | ArrayRef<llvm::Value *> llvmOperands(args); |
| 98 | |
| 99 | SmallVector<llvm::OperandBundleUse> llvmOpBundles; |
| 100 | llvmOpBundles.reserve(N: inst->getNumOperandBundles()); |
| 101 | for (unsigned i = 0; i < inst->getNumOperandBundles(); ++i) |
| 102 | llvmOpBundles.push_back(Elt: inst->getOperandBundleAt(Index: i)); |
| 103 | |
| 104 | // Route `bar.sync` intrinsics to `nvvm.barrier`. |
| 105 | if (isBarrierSyncIntrinsic(id: intrinsicID)) |
| 106 | return convertBarrierSyncIntrinsic(odsBuilder, inst, moduleImport, |
| 107 | llvmOperands, llvmOpBundles); |
| 108 | |
| 109 | #include "mlir/Dialect/LLVMIR/NVVMFromLLVMIRConversions.inc" |
| 110 | } |
| 111 | |
| 112 | return failure(); |
| 113 | } |
| 114 | |
| 115 | namespace { |
| 116 | |
| 117 | /// Implementation of the dialect interface that converts operations belonging |
| 118 | /// to the NVVM dialect. |
| 119 | class NVVMDialectLLVMIRImportInterface : public LLVMImportDialectInterface { |
| 120 | public: |
| 121 | using LLVMImportDialectInterface::LLVMImportDialectInterface; |
| 122 | |
| 123 | /// Converts the LLVM intrinsic to an MLIR NVVM dialect operation if a |
| 124 | /// conversion exits. Returns failure otherwise. |
| 125 | LogicalResult convertIntrinsic(OpBuilder &builder, llvm::CallInst *inst, |
| 126 | LLVM::ModuleImport &moduleImport) const final { |
| 127 | return convertIntrinsicImpl(odsBuilder&: builder, inst, moduleImport); |
| 128 | } |
| 129 | |
| 130 | /// Returns the list of LLVM IR intrinsic identifiers that are convertible to |
| 131 | /// MLIR NVVM dialect intrinsics. |
| 132 | ArrayRef<unsigned> getSupportedIntrinsics() const final { |
| 133 | return getSupportedIntrinsicsImpl(); |
| 134 | } |
| 135 | }; |
| 136 | |
| 137 | } // namespace |
| 138 | |
| 139 | void mlir::registerNVVMDialectImport(DialectRegistry ®istry) { |
| 140 | registry.insert<NVVM::NVVMDialect>(); |
| 141 | registry.addExtension(extensionFn: +[](MLIRContext *ctx, NVVM::NVVMDialect *dialect) { |
| 142 | dialect->addInterfaces<NVVMDialectLLVMIRImportInterface>(); |
| 143 | }); |
| 144 | } |
| 145 | |
| 146 | void mlir::registerNVVMDialectImport(MLIRContext &context) { |
| 147 | DialectRegistry registry; |
| 148 | registerNVVMDialectImport(registry); |
| 149 | context.appendDialectRegistry(registry); |
| 150 | } |
| 151 | |