| 1 | //=====-- NVPTXSubtarget.h - Define Subtarget for the NVPTX ---*- C++ -*--====// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file declares the NVPTX specific subclass of TargetSubtarget. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #ifndef LLVM_LIB_TARGET_NVPTX_NVPTXSUBTARGET_H |
| 14 | #define LLVM_LIB_TARGET_NVPTX_NVPTXSUBTARGET_H |
| 15 | |
| 16 | #include "NVPTX.h" |
| 17 | #include "NVPTXFrameLowering.h" |
| 18 | #include "NVPTXISelLowering.h" |
| 19 | #include "NVPTXInstrInfo.h" |
| 20 | #include "NVPTXRegisterInfo.h" |
| 21 | #include "llvm/CodeGen/TargetSubtargetInfo.h" |
| 22 | #include "llvm/IR/DataLayout.h" |
| 23 | #include "llvm/IR/NVVMIntrinsicUtils.h" |
| 24 | #include "llvm/Support/NVPTXAddrSpace.h" |
| 25 | #include "llvm/TargetParser/NVPTXTargetParser.h" |
| 26 | #include <optional> |
| 27 | |
| 28 | #define OPTIONS_STRUCT_DECL |
| 29 | #include "NVPTXOptions.inc" |
| 30 | |
| 31 | #define |
| 32 | #include "NVPTXGenSubtargetInfo.inc" |
| 33 | |
| 34 | namespace llvm { |
| 35 | |
| 36 | class Function; |
| 37 | class LLVMContext; |
| 38 | class Module; |
| 39 | |
| 40 | const NVPTXOptions &getNVPTXOptions(const LLVMContext &C); |
| 41 | const NVPTXOptions &getNVPTXOptions(const Module &M); |
| 42 | const NVPTXOptions &getNVPTXOptions(const Function &F); |
| 43 | |
| 44 | class NVPTXSubtarget : public NVPTXGenSubtargetInfo { |
| 45 | virtual void anchor(); |
| 46 | |
| 47 | const NVPTXOptions &CLOpts; |
| 48 | |
| 49 | // PTX version x.y is represented as 10*x+y, e.g. 3.1 == 31 |
| 50 | unsigned PTXVersion; |
| 51 | |
| 52 | NVPTX::GPUKind Arch = NVPTX::GK_NONE; |
| 53 | |
| 54 | // Set by every architecture feature. Their bits are the point, so this is |
| 55 | // only here because a subtarget feature must name a field. |
| 56 | bool HasArchitecture = false; |
| 57 | |
| 58 | NVPTXInstrInfo InstrInfo; |
| 59 | NVPTXTargetLowering TLInfo; |
| 60 | std::unique_ptr<const SelectionDAGTargetInfo> TSInfo; |
| 61 | |
| 62 | // NVPTX does not have any call stack frame, but need a NVPTX specific |
| 63 | // FrameLowering class because TargetFrameLowering is abstract. |
| 64 | NVPTXFrameLowering FrameLowering; |
| 65 | |
| 66 | public: |
| 67 | /// This constructor initializes the data members to match that |
| 68 | /// of the specified module. |
| 69 | /// |
| 70 | NVPTXSubtarget(const Triple &TT, StringRef CPU, StringRef FS, |
| 71 | const NVPTXTargetMachine &TM); |
| 72 | |
| 73 | ~NVPTXSubtarget() override; |
| 74 | |
| 75 | const NVPTXOptions &getCLOpts() const { return CLOpts; } |
| 76 | |
| 77 | const TargetFrameLowering *getFrameLowering() const override { |
| 78 | return &FrameLowering; |
| 79 | } |
| 80 | const NVPTXInstrInfo *getInstrInfo() const override { return &InstrInfo; } |
| 81 | const NVPTXRegisterInfo *getRegisterInfo() const override { |
| 82 | return &InstrInfo.getRegisterInfo(); |
| 83 | } |
| 84 | const NVPTXTargetLowering *getTargetLowering() const override { |
| 85 | return &TLInfo; |
| 86 | } |
| 87 | |
| 88 | const SelectionDAGTargetInfo *getSelectionDAGInfo() const override { |
| 89 | return TSInfo.get(); |
| 90 | } |
| 91 | |
| 92 | // True when any of `Features` is enabled. |
| 93 | bool hasAnyFeature(ArrayRef<unsigned> Features) const { |
| 94 | return llvm::any_of(Range&: Features, P: [this](unsigned F) { return hasFeature(Feature: F); }); |
| 95 | } |
| 96 | |
| 97 | bool has256BitVectorLoadStore(unsigned AS) const { |
| 98 | return hasFeature(Feature: NVPTX::SM100) && hasFeature(Feature: NVPTX::PTX88) && |
| 99 | AS == NVPTXAS::ADDRESS_SPACE_GLOBAL; |
| 100 | } |
| 101 | bool hasUsedBytesMaskPragma() const { |
| 102 | return hasFeature(Feature: NVPTX::SM50) && hasFeature(Feature: NVPTX::PTX83); |
| 103 | } |
| 104 | bool hasABIPreserve() const { |
| 105 | return hasFeature(Feature: NVPTX::SM80) && hasFeature(Feature: NVPTX::PTX90); |
| 106 | } |
| 107 | bool hasAtomAddF64() const { return hasFeature(Feature: NVPTX::SM60); } |
| 108 | bool hasAtomScope() const { return hasFeature(Feature: NVPTX::SM60); } |
| 109 | bool hasAtomMinMaxAndOrXor() const { return hasFeature(Feature: NVPTX::SM32); } |
| 110 | bool hasAtomCas16() const { |
| 111 | return hasFeature(Feature: NVPTX::SM70) && hasFeature(Feature: NVPTX::PTX63); |
| 112 | } |
| 113 | bool hasAtomSwap128() const { |
| 114 | return hasFeature(Feature: NVPTX::SM90) && hasFeature(Feature: NVPTX::PTX83); |
| 115 | } |
| 116 | bool hasLdStB128() const { |
| 117 | return hasFeature(Feature: NVPTX::SM70) && hasFeature(Feature: NVPTX::PTX83); |
| 118 | } |
| 119 | bool hasSysScopeLdStB128() const { |
| 120 | return hasLdStB128() && hasFeature(Feature: NVPTX::PTX84); |
| 121 | } |
| 122 | bool hasClusters() const { return hasFeature(Feature: NVPTX::SM90); } |
| 123 | bool hasLDG() const { return hasFeature(Feature: NVPTX::SM32); } |
| 124 | bool hasHWROT32() const { return hasFeature(Feature: NVPTX::SM32); } |
| 125 | bool hasBrx() const { |
| 126 | return hasFeature(Feature: NVPTX::SM30) && hasFeature(Feature: NVPTX::PTX60); |
| 127 | } |
| 128 | bool hasFP16Math() const { return hasFeature(Feature: NVPTX::SM53); } |
| 129 | bool hasBF16Math() const { return hasFeature(Feature: NVPTX::SM80); } |
| 130 | bool allowFP16Math() const; |
| 131 | bool hasMaskOperator() const { return hasFeature(Feature: NVPTX::PTX71); } |
| 132 | bool hasNoReturn() const { |
| 133 | return hasFeature(Feature: NVPTX::SM30) && hasFeature(Feature: NVPTX::PTX64); |
| 134 | } |
| 135 | // Does SM & PTX support memory orderings (weak and atomic: relaxed, acquire, |
| 136 | // release, acq_rel, sc) ? |
| 137 | bool hasMemoryOrdering() const { return hasFeature(Feature: NVPTX::SM70); } |
| 138 | // Does SM & PTX support .acquire and .release qualifiers for fence? |
| 139 | bool hasSplitAcquireAndReleaseFences() const { |
| 140 | return hasFeature(Feature: NVPTX::SM90) && hasFeature(Feature: NVPTX::PTX86); |
| 141 | } |
| 142 | // Does SM & PTX support atomic relaxed MMIO operations ? |
| 143 | bool hasRelaxedMMIO() const { |
| 144 | return hasFeature(Feature: NVPTX::SM70) && hasFeature(Feature: NVPTX::PTX82); |
| 145 | } |
| 146 | bool hasLocalVolatile() const { return hasFeature(Feature: NVPTX::PTX91); } |
| 147 | bool hasDotInstructions() const { return hasFeature(Feature: NVPTX::SM61); } |
| 148 | bool hasCLMAD() const { |
| 149 | return hasFeature(Feature: NVPTX::SM80) && hasFeature(Feature: NVPTX::PTX93); |
| 150 | } |
| 151 | // Cache hint SM/PTX version requirements |
| 152 | bool hasL1EvictionHint() const { |
| 153 | return hasFeature(Feature: NVPTX::SM70) && hasFeature(Feature: NVPTX::PTX74); |
| 154 | } |
| 155 | bool hasL2EvictionHint() const { |
| 156 | return hasFeature(Feature: NVPTX::SM100) && hasFeature(Feature: NVPTX::PTX88); |
| 157 | } |
| 158 | bool hasL2Prefetch64B() const { |
| 159 | return hasFeature(Feature: NVPTX::SM75) && hasFeature(Feature: NVPTX::PTX74); |
| 160 | } |
| 161 | bool hasL2Prefetch128B() const { |
| 162 | return hasFeature(Feature: NVPTX::SM75) && hasFeature(Feature: NVPTX::PTX74); |
| 163 | } |
| 164 | bool hasL2Prefetch256B() const { |
| 165 | return hasFeature(Feature: NVPTX::SM80) && hasFeature(Feature: NVPTX::PTX74); |
| 166 | } |
| 167 | bool hasL2CacheHint() const { |
| 168 | return hasFeature(Feature: NVPTX::SM80) && hasFeature(Feature: NVPTX::PTX74); |
| 169 | } |
| 170 | |
| 171 | // Checks following instructions support: |
| 172 | // - tcgen05.ld/st |
| 173 | // - tcgen05.alloc/dealloc/relinquish |
| 174 | // - tcgen05.cp |
| 175 | // - tcgen05.fence/wait |
| 176 | // - tcgen05.commit |
| 177 | // - tcgen05.mma |
| 178 | bool hasTcgen05InstSupport() const { |
| 179 | return hasAnyFeature(Features: {NVPTX::SM100f, NVPTX::SM110f}); |
| 180 | } |
| 181 | |
| 182 | // f32x2 instructions in Blackwell family |
| 183 | bool hasF32x2Instructions() const; |
| 184 | |
| 185 | bool hasTensormapReplaceSupport() const { |
| 186 | return hasAnyFeature(Features: {NVPTX::SM100f, NVPTX::SM110f, NVPTX::SM120f}) || |
| 187 | (hasFeature(Feature: NVPTX::PTX83) && hasAnyFeature(Features: {NVPTX::SM90a})); |
| 188 | } |
| 189 | |
| 190 | bool hasTensormapReplaceElemtypeSupport(unsigned ElemType) const { |
| 191 | if (ElemType >= static_cast<unsigned>(nvvm::TensormapElemType::B4x16)) |
| 192 | return (hasFeature(Feature: NVPTX::PTX88) && |
| 193 | hasAnyFeature(Features: {NVPTX::SM100f, NVPTX::SM110f, NVPTX::SM120f})) || |
| 194 | (hasFeature(Feature: NVPTX::PTX87) && |
| 195 | hasAnyFeature(Features: {NVPTX::SM100a, NVPTX::SM110a, NVPTX::SM120a})); |
| 196 | |
| 197 | return hasTensormapReplaceSupport(); |
| 198 | } |
| 199 | |
| 200 | bool hasTensormapReplaceSwizzleModeSupport(unsigned SwizzleMode) const { |
| 201 | if (SwizzleMode == |
| 202 | static_cast<unsigned>(nvvm::TensormapSwizzleMode::SWIZZLE_96B)) |
| 203 | return hasAnyFeature(Features: {NVPTX::SM103a}); |
| 204 | |
| 205 | return hasTensormapReplaceSupport(); |
| 206 | } |
| 207 | |
| 208 | // Prior to CUDA 12.3 ptxas did not recognize that the trap instruction |
| 209 | // terminates a basic block. Instead, it would assume that control flow |
| 210 | // continued to the next instruction. The next instruction could be in the |
| 211 | // block that's lexically below it. This would lead to a phantom CFG edges |
| 212 | // being created within ptxas. This issue was fixed in CUDA 12.3. Thus, when |
| 213 | // PTX ISA versions 8.3+ we can confidently say that the bug will not be |
| 214 | // present. |
| 215 | bool hasPTXASUnreachableBug() const { return !hasFeature(Feature: NVPTX::PTX83); } |
| 216 | bool hasCvtaParam() const { |
| 217 | return hasFeature(Feature: NVPTX::SM70) && hasFeature(Feature: NVPTX::PTX77); |
| 218 | } |
| 219 | // The compute capability as a number, for __CUDA_ARCH__. This is the one |
| 220 | // place an architecture needs to be a number, and it is not an identity: |
| 221 | // sm_100, sm_100f and sm_100a all report 100. |
| 222 | unsigned getSmVersion() const { return NVPTX::getSmVersion(Kind: Arch) / 10; } |
| 223 | |
| 224 | // Whether -mcpu named a target at all, as opposed to falling back to the |
| 225 | // default architecture. |
| 226 | bool hasTargetName() const { return !getCPU().empty(); } |
| 227 | |
| 228 | // The architecture's name, which is what `.target` is emitted from. |
| 229 | StringRef getTargetName() const; |
| 230 | |
| 231 | bool hasNativeBF16Support(unsigned Opcode) const; |
| 232 | |
| 233 | // Get maximum value of required alignments among the supported data types. |
| 234 | // From the PTX ISA doc, section 8.2.3: |
| 235 | // The memory consistency model relates operations executed on memory |
| 236 | // locations with scalar data-types, which have a maximum size and alignment |
| 237 | // of 64 bits. Memory operations with a vector data-type are modelled as a |
| 238 | // set of equivalent memory operations with a scalar data-type, executed in |
| 239 | // an unspecified order on the elements in the vector. |
| 240 | unsigned getMaxRequiredAlignment() const { return 8; } |
| 241 | // Get the smallest cmpxchg word size that the hardware supports. |
| 242 | unsigned getMinCmpXchgSizeInBits() const { return 32; } |
| 243 | |
| 244 | unsigned getPTXVersion() const { return PTXVersion; } |
| 245 | |
| 246 | NVPTXSubtarget &initializeSubtargetDependencies(StringRef CPU, StringRef FS); |
| 247 | void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS); |
| 248 | }; |
| 249 | |
| 250 | } // namespace llvm |
| 251 | |
| 252 | #endif |
| 253 | |