| 1 | //===- ROCDLToLLVMIRTranslation.cpp - Translate ROCDL to LLVM IR ----------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements a translation between the MLIR ROCDL dialect and |
| 10 | // LLVM IR. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Target/LLVMIR/Dialect/ROCDL/ROCDLToLLVMIRTranslation.h" |
| 15 | #include "mlir/Dialect/LLVMIR/ROCDLDialect.h" |
| 16 | #include "mlir/IR/BuiltinAttributes.h" |
| 17 | #include "mlir/IR/Operation.h" |
| 18 | #include "mlir/Target/LLVMIR/ModuleTranslation.h" |
| 19 | |
| 20 | #include "llvm/IR/IRBuilder.h" |
| 21 | #include "llvm/IR/IntrinsicsAMDGPU.h" |
| 22 | #include "llvm/IR/LLVMContext.h" |
| 23 | #include "llvm/Support/raw_ostream.h" |
| 24 | #include <cstdint> |
| 25 | |
| 26 | using namespace mlir; |
| 27 | using namespace mlir::LLVM; |
| 28 | using mlir::LLVM::detail::createIntrinsicCall; |
| 29 | |
| 30 | namespace { |
| 31 | /// Implementation of the dialect interface that converts operations belonging |
| 32 | /// to the ROCDL dialect to LLVM IR. |
| 33 | class ROCDLDialectLLVMIRTranslationInterface |
| 34 | : public LLVMTranslationDialectInterface { |
| 35 | public: |
| 36 | using LLVMTranslationDialectInterface::LLVMTranslationDialectInterface; |
| 37 | |
| 38 | /// Translates the given operation to LLVM IR using the provided IR builder |
| 39 | /// and saving the state in `moduleTranslation`. |
| 40 | LogicalResult |
| 41 | convertOperation(Operation *op, llvm::IRBuilderBase &builder, |
| 42 | LLVM::ModuleTranslation &moduleTranslation) const final { |
| 43 | Operation &opInst = *op; |
| 44 | #include "mlir/Dialect/LLVMIR/ROCDLConversions.inc" |
| 45 | |
| 46 | return failure(); |
| 47 | } |
| 48 | |
| 49 | /// Attaches module-level metadata for functions marked as kernels. |
| 50 | LogicalResult |
| 51 | amendOperation(Operation *op, ArrayRef<llvm::Instruction *> instructions, |
| 52 | NamedAttribute attribute, |
| 53 | LLVM::ModuleTranslation &moduleTranslation) const final { |
| 54 | auto *dialect = dyn_cast<ROCDL::ROCDLDialect>(Val: attribute.getNameDialect()); |
| 55 | llvm::LLVMContext &llvmContext = moduleTranslation.getLLVMContext(); |
| 56 | if (dialect->getKernelAttrHelper().getName() == attribute.getName()) { |
| 57 | auto func = dyn_cast<LLVM::LLVMFuncOp>(Val: op); |
| 58 | if (!func) |
| 59 | return op->emitOpError(message: Twine(attribute.getName()) + |
| 60 | " is only supported on `llvm.func` operations" ); |
| 61 | ; |
| 62 | |
| 63 | // For GPU kernels, |
| 64 | // 1. Insert AMDGPU_KERNEL calling convention. |
| 65 | // 2. Insert amdgpu-flat-work-group-size(1, 256) attribute unless the user |
| 66 | // has overriden this value - 256 is the default in clang |
| 67 | llvm::Function *llvmFunc = |
| 68 | moduleTranslation.lookupFunction(name: func.getName()); |
| 69 | llvmFunc->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL); |
| 70 | if (!llvmFunc->hasFnAttribute(Kind: "amdgpu-flat-work-group-size" )) { |
| 71 | llvmFunc->addFnAttr(Kind: "amdgpu-flat-work-group-size" , Val: "1,256" ); |
| 72 | } |
| 73 | |
| 74 | // MLIR's GPU kernel APIs all assume and produce uniformly-sized |
| 75 | // workgroups, so the lowering of the `rocdl.kernel` marker encodes this |
| 76 | // assumption. This assumption may be overridden by setting |
| 77 | // `rocdl.uniform_work_group_size` on a given function. |
| 78 | if (!llvmFunc->hasFnAttribute(Kind: "uniform-work-group-size" )) |
| 79 | llvmFunc->addFnAttr(Kind: "uniform-work-group-size" ); |
| 80 | } |
| 81 | // Override flat-work-group-size |
| 82 | // TODO: update clients to rocdl.flat_work_group_size instead, |
| 83 | // then remove this half of the branch |
| 84 | if (dialect->getMaxFlatWorkGroupSizeAttrHelper().getName() == |
| 85 | attribute.getName()) { |
| 86 | auto func = dyn_cast<LLVM::LLVMFuncOp>(Val: op); |
| 87 | if (!func) |
| 88 | return op->emitOpError(message: Twine(attribute.getName()) + |
| 89 | " is only supported on `llvm.func` operations" ); |
| 90 | auto value = dyn_cast<IntegerAttr>(Val: attribute.getValue()); |
| 91 | if (!value) |
| 92 | return op->emitOpError(message: Twine(attribute.getName()) + |
| 93 | " must be an integer" ); |
| 94 | |
| 95 | llvm::Function *llvmFunc = |
| 96 | moduleTranslation.lookupFunction(name: func.getName()); |
| 97 | llvm::SmallString<8> llvmAttrValue; |
| 98 | llvm::raw_svector_ostream attrValueStream(llvmAttrValue); |
| 99 | attrValueStream << "1," << value.getInt(); |
| 100 | llvmFunc->addFnAttr(Kind: "amdgpu-flat-work-group-size" , Val: llvmAttrValue); |
| 101 | } |
| 102 | if (dialect->getWavesPerEuAttrHelper().getName() == attribute.getName()) { |
| 103 | auto func = dyn_cast<LLVM::LLVMFuncOp>(Val: op); |
| 104 | if (!func) |
| 105 | return op->emitOpError(message: Twine(attribute.getName()) + |
| 106 | " is only supported on `llvm.func` operations" ); |
| 107 | |
| 108 | llvm::Function *llvmFunc = |
| 109 | moduleTranslation.lookupFunction(name: func.getName()); |
| 110 | llvm::SmallString<8> llvmAttrValue; |
| 111 | if (auto strValue = dyn_cast<StringAttr>(Val: attribute.getValue())) { |
| 112 | llvmAttrValue.append(RHS: strValue.getValue()); |
| 113 | } else if (auto intValue = dyn_cast<IntegerAttr>(Val: attribute.getValue())) { |
| 114 | llvm::raw_svector_ostream attrValueStream(llvmAttrValue); |
| 115 | attrValueStream << intValue.getInt(); |
| 116 | } else { |
| 117 | return op->emitOpError(message: Twine(attribute.getName()) + |
| 118 | " must be a string or an integer" ); |
| 119 | } |
| 120 | llvmFunc->addFnAttr(Kind: "amdgpu-waves-per-eu" , Val: llvmAttrValue); |
| 121 | } |
| 122 | if (dialect->getFlatWorkGroupSizeAttrHelper().getName() == |
| 123 | attribute.getName()) { |
| 124 | auto func = dyn_cast<LLVM::LLVMFuncOp>(Val: op); |
| 125 | if (!func) |
| 126 | return op->emitOpError(message: Twine(attribute.getName()) + |
| 127 | " is only supported on `llvm.func` operations" ); |
| 128 | auto value = dyn_cast<StringAttr>(Val: attribute.getValue()); |
| 129 | if (!value) |
| 130 | return op->emitOpError(message: Twine(attribute.getName()) + |
| 131 | " must be a string" ); |
| 132 | |
| 133 | llvm::Function *llvmFunc = |
| 134 | moduleTranslation.lookupFunction(name: func.getName()); |
| 135 | llvm::SmallString<8> llvmAttrValue; |
| 136 | llvmAttrValue.append(RHS: value.getValue()); |
| 137 | llvmFunc->addFnAttr(Kind: "amdgpu-flat-work-group-size" , Val: llvmAttrValue); |
| 138 | } |
| 139 | if (ROCDL::ROCDLDialect::getUniformWorkGroupSizeAttrName() == |
| 140 | attribute.getName()) { |
| 141 | auto func = dyn_cast<LLVM::LLVMFuncOp>(Val: op); |
| 142 | if (!func) |
| 143 | return op->emitOpError(message: Twine(attribute.getName()) + |
| 144 | " is only supported on `llvm.func` operations" ); |
| 145 | auto value = dyn_cast<BoolAttr>(Val: attribute.getValue()); |
| 146 | if (!value) |
| 147 | return op->emitOpError(message: Twine(attribute.getName()) + |
| 148 | " must be a boolean" ); |
| 149 | llvm::Function *llvmFunc = |
| 150 | moduleTranslation.lookupFunction(name: func.getName()); |
| 151 | if (value.getValue()) |
| 152 | llvmFunc->addFnAttr(Kind: "uniform-work-group-size" ); |
| 153 | else |
| 154 | llvmFunc->removeFnAttr(Kind: "uniform-work-group-size" ); |
| 155 | } |
| 156 | |
| 157 | bool isXnack = |
| 158 | dialect->getXnackAttrHelper().getName() == attribute.getName(); |
| 159 | bool isSramecc = |
| 160 | dialect->getSrameccAttrHelper().getName() == attribute.getName(); |
| 161 | if (isXnack || isSramecc) { |
| 162 | auto value = dyn_cast<BoolAttr>(Val: attribute.getValue()); |
| 163 | if (!value) |
| 164 | return op->emitOpError(message: Twine(attribute.getName()) + |
| 165 | " must be a boolean" ); |
| 166 | StringRef key = isXnack |
| 167 | ? ROCDL::ROCDLDialect::getModuleFlagKeyXnackName() |
| 168 | : ROCDL::ROCDLDialect::getModuleFlagKeySramEccName(); |
| 169 | moduleTranslation.getLLVMModule()->addModuleFlag( |
| 170 | Behavior: llvm::Module::Error, Key: key, |
| 171 | Val: llvm::ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: llvmContext), |
| 172 | V: value.getValue())); |
| 173 | } |
| 174 | if (dialect->getUnsafeFpAtomicsAttrHelper().getName() == |
| 175 | attribute.getName()) { |
| 176 | auto func = dyn_cast<LLVM::LLVMFuncOp>(Val: op); |
| 177 | if (!func) |
| 178 | return op->emitOpError(message: Twine(attribute.getName()) + |
| 179 | " is only supported on `llvm.func` operations" ); |
| 180 | auto value = dyn_cast<BoolAttr>(Val: attribute.getValue()); |
| 181 | if (!value) |
| 182 | return op->emitOpError(message: Twine(attribute.getName()) + |
| 183 | " must be a boolean" ); |
| 184 | llvm::Function *llvmFunc = |
| 185 | moduleTranslation.lookupFunction(name: func.getName()); |
| 186 | llvmFunc->addFnAttr(Kind: "amdgpu-unsafe-fp-atomics" , |
| 187 | Val: value.getValue() ? "true" : "false" ); |
| 188 | } |
| 189 | // Set reqd_work_group_size metadata |
| 190 | if (dialect->getReqdWorkGroupSizeAttrHelper().getName() == |
| 191 | attribute.getName()) { |
| 192 | auto func = dyn_cast<LLVM::LLVMFuncOp>(Val: op); |
| 193 | if (!func) |
| 194 | return op->emitOpError(message: Twine(attribute.getName()) + |
| 195 | " is only supported on `llvm.func` operations" ); |
| 196 | auto value = dyn_cast<DenseI32ArrayAttr>(Val: attribute.getValue()); |
| 197 | if (!value) |
| 198 | return op->emitOpError(message: Twine(attribute.getName()) + |
| 199 | " must be a dense i32 array attribute" ); |
| 200 | if (value.asArrayRef().size() != 3) |
| 201 | return op->emitOpError(message: Twine(attribute.getName()) + |
| 202 | " must contain exactly three values" ); |
| 203 | |
| 204 | uint64_t FlatWorkGroupSize = 1; |
| 205 | SmallVector<llvm::Metadata *, 3> metadata; |
| 206 | llvm::Type *i32 = llvm::IntegerType::get(C&: llvmContext, NumBits: 32); |
| 207 | for (int32_t i : value.asArrayRef()) { |
| 208 | FlatWorkGroupSize *= static_cast<uint32_t>(i); |
| 209 | llvm::Constant *constant = llvm::ConstantInt::get(Ty: i32, V: i); |
| 210 | metadata.push_back(Elt: llvm::ConstantAsMetadata::get(C: constant)); |
| 211 | } |
| 212 | llvm::Function *llvmFunc = |
| 213 | moduleTranslation.lookupFunction(name: func.getName()); |
| 214 | llvm::SmallString<16> expectedFlatWorkGroupSize; |
| 215 | llvm::raw_svector_ostream attrValueStream(expectedFlatWorkGroupSize); |
| 216 | attrValueStream << FlatWorkGroupSize << "," << FlatWorkGroupSize; |
| 217 | |
| 218 | StringRef flatAttrName = |
| 219 | dialect->getFlatWorkGroupSizeAttrHelper().getName(); |
| 220 | if (auto flatAttr = dyn_cast_if_present<StringAttr>( |
| 221 | Val: op->getDiscardableAttr(name: flatAttrName))) { |
| 222 | if (flatAttr.getValue() != expectedFlatWorkGroupSize) |
| 223 | return op->emitOpError(message: Twine(flatAttrName) + |
| 224 | " must match rocdl.reqd_work_group_size" ); |
| 225 | } |
| 226 | |
| 227 | StringRef maxFlatAttrName = |
| 228 | dialect->getMaxFlatWorkGroupSizeAttrHelper().getName(); |
| 229 | if (auto maxFlatAttr = dyn_cast_if_present<IntegerAttr>( |
| 230 | Val: op->getDiscardableAttr(name: maxFlatAttrName))) { |
| 231 | llvm::SmallString<16> expectedMaxFlatWorkGroupSize; |
| 232 | llvm::raw_svector_ostream maxAttrValueStream( |
| 233 | expectedMaxFlatWorkGroupSize); |
| 234 | maxAttrValueStream << "1," << maxFlatAttr.getInt(); |
| 235 | if (expectedMaxFlatWorkGroupSize != expectedFlatWorkGroupSize) |
| 236 | return op->emitOpError(message: Twine(maxFlatAttrName) + |
| 237 | " must match rocdl.reqd_work_group_size" ); |
| 238 | } |
| 239 | |
| 240 | llvmFunc->addFnAttr(Kind: "amdgpu-flat-work-group-size" , |
| 241 | Val: expectedFlatWorkGroupSize); |
| 242 | llvm::MDNode *node = llvm::MDNode::get(Context&: llvmContext, MDs: metadata); |
| 243 | llvmFunc->setMetadata(Kind: "reqd_work_group_size" , Node: node); |
| 244 | } |
| 245 | |
| 246 | // Atomic and nontemporal metadata |
| 247 | if (dialect->getLastUseAttrHelper().getName() == attribute.getName()) { |
| 248 | for (llvm::Instruction *i : instructions) |
| 249 | i->setMetadata(Kind: "amdgpu.last.use" , Node: llvm::MDNode::get(Context&: llvmContext, MDs: {})); |
| 250 | } |
| 251 | if (dialect->getNoRemoteMemoryAttrHelper().getName() == |
| 252 | attribute.getName()) { |
| 253 | for (llvm::Instruction *i : instructions) |
| 254 | i->setMetadata(Kind: "amdgpu.no.remote.memory" , |
| 255 | Node: llvm::MDNode::get(Context&: llvmContext, MDs: {})); |
| 256 | } |
| 257 | if (dialect->getNoFineGrainedMemoryAttrHelper().getName() == |
| 258 | attribute.getName()) { |
| 259 | for (llvm::Instruction *i : instructions) |
| 260 | i->setMetadata(Kind: "amdgpu.no.fine.grained.memory" , |
| 261 | Node: llvm::MDNode::get(Context&: llvmContext, MDs: {})); |
| 262 | } |
| 263 | if (dialect->getIgnoreDenormalModeAttrHelper().getName() == |
| 264 | attribute.getName()) { |
| 265 | op->emitWarning(message: "rocdl.ignore_denormal_mode is deprecated; use " |
| 266 | "ignore_denormal_mode on llvm.atomicrmw instead" ); |
| 267 | for (llvm::Instruction *i : instructions) |
| 268 | i->setMetadata(KindID: llvm::LLVMContext::MD_atomic_ignore_denormal_mode, |
| 269 | Node: llvm::MDNode::get(Context&: llvmContext, MDs: {})); |
| 270 | } |
| 271 | |
| 272 | return success(); |
| 273 | } |
| 274 | }; |
| 275 | } // namespace |
| 276 | |
| 277 | void mlir::registerROCDLDialectTranslation(DialectRegistry ®istry) { |
| 278 | registry.insert<ROCDL::ROCDLDialect>(); |
| 279 | registry.addExtension(extensionFn: +[](MLIRContext *ctx, ROCDL::ROCDLDialect *dialect) { |
| 280 | dialect->addInterfaces<ROCDLDialectLLVMIRTranslationInterface>(); |
| 281 | }); |
| 282 | } |
| 283 | |
| 284 | void mlir::registerROCDLDialectTranslation(MLIRContext &context) { |
| 285 | DialectRegistry registry; |
| 286 | registerROCDLDialectTranslation(registry); |
| 287 | context.appendDialectRegistry(registry); |
| 288 | } |
| 289 | |