| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements lowering of CIR attributes and operations directly to |
| 10 | // LLVMIR. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 15 | #include "mlir/Dialect/LLVMIR/LLVMModuleOps.h" |
| 16 | #include "mlir/IR/DialectRegistry.h" |
| 17 | #include "mlir/Target/LLVMIR/LLVMTranslationInterface.h" |
| 18 | #include "mlir/Target/LLVMIR/ModuleTranslation.h" |
| 19 | #include "clang/CIR/Dialect/IR/CIRDialect.h" |
| 20 | #include "llvm/ADT/ArrayRef.h" |
| 21 | #include "llvm/IR/Constant.h" |
| 22 | #include "llvm/IR/Constants.h" |
| 23 | #include "llvm/IR/GlobalVariable.h" |
| 24 | #include "llvm/IR/MDBuilder.h" |
| 25 | #include "llvm/IR/Metadata.h" |
| 26 | #include "llvm/Support/AMDGPUAddrSpace.h" |
| 27 | |
| 28 | using namespace llvm; |
| 29 | |
| 30 | namespace cir { |
| 31 | namespace direct { |
| 32 | |
| 33 | /// Implementation of the dialect interface that converts CIR attributes to LLVM |
| 34 | /// IR metadata. |
| 35 | class CIRDialectLLVMIRTranslationInterface |
| 36 | : public mlir::LLVMTranslationDialectInterface { |
| 37 | public: |
| 38 | using LLVMTranslationDialectInterface::LLVMTranslationDialectInterface; |
| 39 | |
| 40 | /// Translates the given operation to LLVM IR using the provided IR builder |
| 41 | /// and saving the state in `moduleTranslation`. |
| 42 | mlir::LogicalResult convertOperation( |
| 43 | mlir::Operation *op, llvm::IRBuilderBase &builder, |
| 44 | mlir::LLVM::ModuleTranslation &moduleTranslation) const final { |
| 45 | |
| 46 | if (auto cirOp = llvm::dyn_cast<mlir::LLVM::ZeroOp>(Val: op)) |
| 47 | moduleTranslation.mapValue(value: cirOp.getResult()) = |
| 48 | llvm::Constant::getNullValue( |
| 49 | Ty: moduleTranslation.convertType(type: cirOp.getType())); |
| 50 | |
| 51 | return mlir::success(); |
| 52 | } |
| 53 | |
| 54 | /// Any named attribute in the CIR dialect, i.e, with name started with |
| 55 | /// "cir.", will be handled here. |
| 56 | virtual mlir::LogicalResult amendOperation( |
| 57 | mlir::Operation *op, llvm::ArrayRef<llvm::Instruction *> instructions, |
| 58 | mlir::NamedAttribute attribute, |
| 59 | mlir::LLVM::ModuleTranslation &moduleTranslation) const override { |
| 60 | if (auto func = dyn_cast<mlir::LLVM::LLVMFuncOp>(Val: op)) { |
| 61 | if (mlir::failed( |
| 62 | Result: amendFunction(func, instructions, attribute, moduleTranslation))) |
| 63 | return mlir::failure(); |
| 64 | } else if (auto mod = dyn_cast<mlir::ModuleOp>(Val: op)) { |
| 65 | if (mlir::failed(Result: amendModule(mod, attribute, moduleTranslation))) |
| 66 | return mlir::failure(); |
| 67 | } else if (attribute.getName() == "cir.riscv_nontemporal_domain" ) { |
| 68 | if (mlir::failed(Result: amendRISCVNontemporalDomain(op, instructions, attribute, |
| 69 | moduleTranslation))) |
| 70 | return mlir::failure(); |
| 71 | } else if (attribute.getName() == cir::CIRDialect::getSrcLocAttrName()) { |
| 72 | if (mlir::failed(Result: amendSrcLoc(instructions, attribute, moduleTranslation))) |
| 73 | return mlir::failure(); |
| 74 | } else if (attribute.getName() == |
| 75 | cir::CIRDialect::getAMDGPUNoFineGrainedMemoryAttrName() || |
| 76 | attribute.getName() == |
| 77 | cir::CIRDialect::getAMDGPUNoRemoteMemoryAttrName() || |
| 78 | attribute.getName() == |
| 79 | cir::CIRDialect::getAMDGPUIgnoreDenormalModeAttrName() || |
| 80 | attribute.getName() == |
| 81 | cir::CIRDialect::getAMDGPUNoPrivateAddrSpaceAttrName()) { |
| 82 | amendAMDGPUAtomicMetadata(instructions, attribute, moduleTranslation); |
| 83 | } |
| 84 | return mlir::success(); |
| 85 | } |
| 86 | |
| 87 | private: |
| 88 | // Convert the raw-encoded clang::SourceLocation into !srcloc, so that |
| 89 | // dontcall-error/warn works. |
| 90 | mlir::LogicalResult |
| 91 | amendSrcLoc(llvm::ArrayRef<llvm::Instruction *> instructions, |
| 92 | mlir::NamedAttribute attribute, |
| 93 | mlir::LLVM::ModuleTranslation &moduleTranslation) const { |
| 94 | auto srcLoc = mlir::dyn_cast<mlir::IntegerAttr>(Val: attribute.getValue()); |
| 95 | if (!srcLoc) |
| 96 | return mlir::failure(); |
| 97 | |
| 98 | llvm::LLVMContext &llvmContext = moduleTranslation.getLLVMContext(); |
| 99 | llvm::MDNode *node = llvm::MDNode::get( |
| 100 | Context&: llvmContext, MDs: llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get( |
| 101 | Ty: llvm::Type::getInt64Ty(C&: llvmContext), |
| 102 | V: srcLoc.getValue().getZExtValue()))); |
| 103 | for (llvm::Instruction *inst : instructions) |
| 104 | inst->setMetadata(Kind: "srcloc" , Node: node); |
| 105 | return mlir::success(); |
| 106 | } |
| 107 | |
| 108 | /// Attach the AMDGPU atomic metadata that lets the backend select a native |
| 109 | /// atomic instruction instead of expanding to a cmpxchg loop. |
| 110 | void amendAMDGPUAtomicMetadata( |
| 111 | llvm::ArrayRef<llvm::Instruction *> instructions, |
| 112 | mlir::NamedAttribute attribute, |
| 113 | mlir::LLVM::ModuleTranslation &moduleTranslation) const { |
| 114 | if (attribute.getName() == |
| 115 | cir::CIRDialect::getAMDGPUNoPrivateAddrSpaceAttrName()) { |
| 116 | llvm::MDBuilder mdHelper(moduleTranslation.getLLVMContext()); |
| 117 | llvm::MDNode *asRange = mdHelper.createRange( |
| 118 | Lo: llvm::APInt(32, llvm::AMDGPUAS::PRIVATE_ADDRESS), |
| 119 | Hi: llvm::APInt(32, llvm::AMDGPUAS::PRIVATE_ADDRESS + 1)); |
| 120 | for (llvm::Instruction *inst : instructions) |
| 121 | inst->setMetadata(KindID: llvm::LLVMContext::MD_noalias_addrspace, Node: asRange); |
| 122 | return; |
| 123 | } |
| 124 | llvm::MDNode *empty = |
| 125 | llvm::MDNode::get(Context&: moduleTranslation.getLLVMContext(), MDs: {}); |
| 126 | // !atomic.ignore.denormal.mode is a fixed metadata kind so it has to be |
| 127 | // attached via its enum rather than by name. |
| 128 | if (attribute.getName() == |
| 129 | cir::CIRDialect::getAMDGPUIgnoreDenormalModeAttrName()) { |
| 130 | for (llvm::Instruction *inst : instructions) |
| 131 | inst->setMetadata(KindID: llvm::LLVMContext::MD_atomic_ignore_denormal_mode, |
| 132 | Node: empty); |
| 133 | return; |
| 134 | } |
| 135 | llvm::StringRef mdName; |
| 136 | if (attribute.getName() == |
| 137 | cir::CIRDialect::getAMDGPUNoFineGrainedMemoryAttrName()) |
| 138 | mdName = "amdgpu.no.fine.grained.memory" ; |
| 139 | else if (attribute.getName() == |
| 140 | cir::CIRDialect::getAMDGPUNoRemoteMemoryAttrName()) |
| 141 | mdName = "amdgpu.no.remote.memory" ; |
| 142 | for (llvm::Instruction *inst : instructions) |
| 143 | inst->setMetadata(Kind: mdName, Node: empty); |
| 144 | } |
| 145 | |
| 146 | mlir::LogicalResult amendRISCVNontemporalDomain( |
| 147 | mlir::Operation *op, llvm::ArrayRef<llvm::Instruction *> instructions, |
| 148 | mlir::NamedAttribute attribute, |
| 149 | mlir::LLVM::ModuleTranslation &moduleTranslation) const { |
| 150 | auto domain = mlir::dyn_cast<mlir::IntegerAttr>(Val: attribute.getValue()); |
| 151 | if (!domain) |
| 152 | return op->emitError() |
| 153 | << "expected cir.riscv_nontemporal_domain to be an integer" ; |
| 154 | |
| 155 | llvm::LLVMContext &llvmContext = moduleTranslation.getLLVMContext(); |
| 156 | llvm::MDNode *node = llvm::MDNode::get( |
| 157 | Context&: llvmContext, MDs: llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get( |
| 158 | Ty: llvm::Type::getInt32Ty(C&: llvmContext), |
| 159 | V: domain.getValue().getZExtValue()))); |
| 160 | for (llvm::Instruction *inst : instructions) |
| 161 | inst->setMetadata(Kind: "riscv-nontemporal-domain" , Node: node); |
| 162 | return mlir::success(); |
| 163 | } |
| 164 | |
| 165 | // Translate CIR function attributes to LLVM function attributes. |
| 166 | mlir::LogicalResult |
| 167 | amendFunction(mlir::LLVM::LLVMFuncOp func, |
| 168 | llvm::ArrayRef<llvm::Instruction *> instructions, |
| 169 | mlir::NamedAttribute attribute, |
| 170 | mlir::LLVM::ModuleTranslation &moduleTranslation) const { |
| 171 | llvm::Function *llvmFunc = moduleTranslation.lookupFunction(name: func.getName()); |
| 172 | llvm::StringRef attrName = attribute.getName().strref(); |
| 173 | |
| 174 | // Strip the "cir." prefix to get the LLVM attribute name. |
| 175 | llvm::StringRef llvmAttrName = attrName.substr(Start: strlen(s: "cir." )); |
| 176 | if (auto strAttr = mlir::dyn_cast<mlir::StringAttr>(Val: attribute.getValue())) |
| 177 | llvmFunc->addFnAttr(Kind: llvmAttrName, Val: strAttr.getValue()); |
| 178 | return mlir::success(); |
| 179 | } |
| 180 | |
| 181 | // Translate CIR's module attributes to LLVM's module metadata |
| 182 | mlir::LogicalResult |
| 183 | amendModule(mlir::ModuleOp mod, mlir::NamedAttribute attribute, |
| 184 | mlir::LLVM::ModuleTranslation &moduleTranslation) const { |
| 185 | llvm::Module *llvmModule = moduleTranslation.getLLVMModule(); |
| 186 | llvm::LLVMContext &llvmContext = llvmModule->getContext(); |
| 187 | |
| 188 | if (attribute.getName() == "cir.amdhsa_code_object_version" ) { |
| 189 | if (auto intAttr = |
| 190 | mlir::dyn_cast<mlir::IntegerAttr>(Val: attribute.getValue())) { |
| 191 | llvmModule->addModuleFlag(Behavior: llvm::Module::Error, |
| 192 | Key: "amdhsa_code_object_version" , |
| 193 | Val: static_cast<uint32_t>(intAttr.getInt())); |
| 194 | } |
| 195 | } |
| 196 | |
| 197 | if (attribute.getName() == "cir.amdgpu_printf_kind" ) { |
| 198 | if (auto strAttr = |
| 199 | mlir::dyn_cast<mlir::StringAttr>(Val: attribute.getValue())) { |
| 200 | llvm::MDString *mdStr = |
| 201 | llvm::MDString::get(Context&: llvmContext, Str: strAttr.getValue()); |
| 202 | llvmModule->addModuleFlag(Behavior: llvm::Module::Error, Key: "amdgpu_printf_kind" , |
| 203 | Val: mdStr); |
| 204 | } |
| 205 | } |
| 206 | |
| 207 | if (attribute.getName() == "cir.amdgpu_xnack" ) { |
| 208 | if (auto intAttr = |
| 209 | mlir::dyn_cast<mlir::IntegerAttr>(Val: attribute.getValue())) { |
| 210 | llvmModule->addModuleFlag(Behavior: llvm::Module::Error, Key: "amdgpu.xnack" , |
| 211 | Val: static_cast<uint32_t>(intAttr.getInt())); |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | if (attribute.getName() == "cir.amdgpu_sramecc" ) { |
| 216 | if (auto intAttr = |
| 217 | mlir::dyn_cast<mlir::IntegerAttr>(Val: attribute.getValue())) { |
| 218 | llvmModule->addModuleFlag(Behavior: llvm::Module::Error, Key: "amdgpu.sramecc" , |
| 219 | Val: static_cast<uint32_t>(intAttr.getInt())); |
| 220 | } |
| 221 | } |
| 222 | |
| 223 | return mlir::success(); |
| 224 | } |
| 225 | }; |
| 226 | |
| 227 | void registerCIRDialectTranslation(mlir::DialectRegistry ®istry) { |
| 228 | registry.insert<cir::CIRDialect>(); |
| 229 | registry.addExtension(extensionFn: +[](mlir::MLIRContext *ctx, cir::CIRDialect *dialect) { |
| 230 | dialect->addInterfaces<CIRDialectLLVMIRTranslationInterface>(); |
| 231 | }); |
| 232 | } |
| 233 | |
| 234 | } // namespace direct |
| 235 | } // namespace cir |
| 236 | |
| 237 | namespace mlir { |
| 238 | void registerCIRDialectTranslation(mlir::MLIRContext &context) { |
| 239 | mlir::DialectRegistry registry; |
| 240 | cir::direct::registerCIRDialectTranslation(registry); |
| 241 | context.appendDialectRegistry(registry); |
| 242 | } |
| 243 | } // namespace mlir |
| 244 | |