| 1 | //===---- NVPTX.cpp - NVPTX-specific CIR CodeGen --------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This provides NVPTX-specific CIR CodeGen logic. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "../ABIInfo.h" |
| 14 | #include "../CIRGenModule.h" |
| 15 | #include "../TargetInfo.h" |
| 16 | |
| 17 | #include "clang/CIR/Dialect/IR/CIRTypes.h" |
| 18 | #include "llvm/ADT/StringExtras.h" |
| 19 | #include "llvm/Support/NVVMAttributes.h" |
| 20 | |
| 21 | using namespace clang; |
| 22 | using namespace clang::CIRGen; |
| 23 | |
| 24 | namespace { |
| 25 | |
| 26 | /// Handle the launch_bounds attribute, which maps onto the nvvm.maxntid, |
| 27 | /// nvvm.minctasm and nvvm.maxclusterrank function attributes. |
| 28 | static void handleCUDALaunchBoundsAttr(const CUDALaunchBoundsAttr *attr, |
| 29 | cir::FuncOp func, CIRGenModule &cgm, |
| 30 | CIRGenBuilderTy &builder) { |
| 31 | auto setNVVMAttr = [&](llvm::StringRef name, const llvm::APSInt &value) { |
| 32 | func->setAttr(name: ("cir." + name).str(), |
| 33 | value: builder.getStringAttr(bytes: llvm::utostr(X: value.getExtValue()))); |
| 34 | }; |
| 35 | |
| 36 | llvm::APSInt maxThreads(32); |
| 37 | maxThreads = |
| 38 | attr->getMaxThreads()->EvaluateKnownConstInt(Ctx: cgm.getASTContext()); |
| 39 | if (maxThreads > 0) |
| 40 | setNVVMAttr(llvm::NVVMAttr::MaxNTID, maxThreads); |
| 41 | |
| 42 | // min and max blocks is an optional argument for CUDALaunchBoundsAttr. If it |
| 43 | // was not specified in __launch_bounds__ or if the user specified a 0 value, |
| 44 | // we don't have to add a PTX directive. |
| 45 | if (attr->getMinBlocks()) { |
| 46 | llvm::APSInt minBlocks(32); |
| 47 | minBlocks = |
| 48 | attr->getMinBlocks()->EvaluateKnownConstInt(Ctx: cgm.getASTContext()); |
| 49 | if (minBlocks > 0) |
| 50 | setNVVMAttr(llvm::NVVMAttr::MinCTASm, minBlocks); |
| 51 | } |
| 52 | |
| 53 | if (attr->getMaxBlocks()) { |
| 54 | llvm::APSInt maxBlocks(32); |
| 55 | maxBlocks = |
| 56 | attr->getMaxBlocks()->EvaluateKnownConstInt(Ctx: cgm.getASTContext()); |
| 57 | if (maxBlocks > 0) |
| 58 | setNVVMAttr(llvm::NVVMAttr::MaxClusterRank, maxBlocks); |
| 59 | } |
| 60 | } |
| 61 | |
| 62 | class NVPTXABIInfo : public ABIInfo { |
| 63 | public: |
| 64 | NVPTXABIInfo(CIRGenTypes &cgt) : ABIInfo(cgt) {} |
| 65 | }; |
| 66 | |
| 67 | class NVPTXTargetCIRGenInfo : public TargetCIRGenInfo { |
| 68 | public: |
| 69 | NVPTXTargetCIRGenInfo(CIRGenTypes &cgt) |
| 70 | : TargetCIRGenInfo(std::make_unique<NVPTXABIInfo>(args&: cgt)) {} |
| 71 | |
| 72 | void setTargetAttributes(const clang::Decl *decl, mlir::Operation *global, |
| 73 | CIRGenModule &cgm) const override { |
| 74 | auto globalValue = mlir::dyn_cast<cir::CIRGlobalValueInterface>(Val: global); |
| 75 | if (globalValue && globalValue.isDeclaration()) |
| 76 | return; |
| 77 | |
| 78 | const auto *vd = dyn_cast_or_null<VarDecl>(Val: decl); |
| 79 | if (vd) { |
| 80 | if (cgm.getLangOpts().CUDA) { |
| 81 | if (vd->getType()->isCUDADeviceBuiltinSurfaceType() || |
| 82 | vd->getType()->isCUDADeviceBuiltinTextureType()) |
| 83 | assert(!cir::MissingFeatures::emitNVVMMetadata()); |
| 84 | return; |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | const auto *fd = dyn_cast_or_null<FunctionDecl>(Val: decl); |
| 89 | if (!fd) |
| 90 | return; |
| 91 | |
| 92 | auto func = mlir::cast<cir::FuncOp>(Val: global); |
| 93 | |
| 94 | // Perform special handling in OpenCL/CUDA mode. |
| 95 | if (cgm.getLangOpts().OpenCL || cgm.getLangOpts().CUDA) { |
| 96 | // Use function attributes to check for kernel functions. By default, all |
| 97 | // functions are device functions. |
| 98 | if (fd->hasAttr<DeviceKernelAttr>() || fd->hasAttr<CUDAGlobalAttr>()) { |
| 99 | // OpenCL/CUDA kernel functions get kernel metadata. Kernel functions |
| 100 | // are also not subject to inlining. |
| 101 | func.setInlineKind(cir::InlineKind::NoInline); |
| 102 | if (fd->hasAttr<CUDAGlobalAttr>()) { |
| 103 | func.setCallingConv(getDeviceKernelCallingConv()); |
| 104 | assert(!cir::MissingFeatures::opFuncParameterAttributes()); |
| 105 | } |
| 106 | if (const auto *attr = fd->getAttr<CUDALaunchBoundsAttr>()) |
| 107 | handleCUDALaunchBoundsAttr(attr, func, cgm, builder&: cgm.getBuilder()); |
| 108 | } |
| 109 | } |
| 110 | } |
| 111 | |
| 112 | cir::CallingConv getDeviceKernelCallingConv() const override { |
| 113 | return cir::CallingConv::PTXKernel; |
| 114 | } |
| 115 | |
| 116 | mlir::Type getCUDADeviceBuiltinSurfaceDeviceType() const override { |
| 117 | return cir::CUDADeviceSurfaceType::get(ctx: &getABIInfo().cgt.getMLIRContext()); |
| 118 | } |
| 119 | |
| 120 | mlir::Type getCUDADeviceBuiltinTextureDeviceType() const override { |
| 121 | return cir::CUDADeviceTextureType::get(ctx: &getABIInfo().cgt.getMLIRContext()); |
| 122 | } |
| 123 | |
| 124 | // A null pointer in the local address space is materialized via an address |
| 125 | // space cast from null in the generic address space, so that it converts |
| 126 | // back to a generic null pointer. |
| 127 | mlir::Value getNullPointer(CIRGenModule &cgm, cir::PointerType ptrTy, |
| 128 | QualType qt, mlir::Location loc) const override { |
| 129 | CIRGenBuilderTy &builder = cgm.getBuilder(); |
| 130 | // The local address space is either a language address space or an |
| 131 | // explicit target address space at this point. |
| 132 | mlir::ptr::MemorySpaceAttrInterface addrSpace = ptrTy.getAddrSpace(); |
| 133 | auto targetAS = |
| 134 | mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>(Val&: addrSpace); |
| 135 | bool isLocal = |
| 136 | cir::isMatchingAddressSpace(cirAS: addrSpace, as: LangAS::opencl_local) || |
| 137 | (targetAS && |
| 138 | targetAS.getValue() == |
| 139 | cgm.getASTContext().getTargetAddressSpace(AS: LangAS::opencl_local)); |
| 140 | if (!isLocal) |
| 141 | return builder.getNullPtr(ty: ptrTy, loc); |
| 142 | |
| 143 | cir::PointerType genericPtrTy = |
| 144 | builder.getPointerTo(ty: ptrTy.getPointee(), langAS: LangAS::opencl_generic); |
| 145 | return builder.createAddrSpaceCast( |
| 146 | loc, src: builder.getNullPtr(ty: genericPtrTy, loc), newTy: ptrTy); |
| 147 | } |
| 148 | }; |
| 149 | |
| 150 | } // namespace |
| 151 | |
| 152 | std::unique_ptr<TargetCIRGenInfo> |
| 153 | clang::CIRGen::createNVPTXTargetCIRGenInfo(CIRGenTypes &cgt) { |
| 154 | return std::make_unique<NVPTXTargetCIRGenInfo>(args&: cgt); |
| 155 | } |
| 156 | |