| 1 | //===-- NVPTXTargetMachine.cpp - Define TargetMachine for NVPTX -----------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Top-level implementation for the NVPTX target. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "NVPTXTargetMachine.h" |
| 14 | #include "NVPTX.h" |
| 15 | #include "NVPTXAliasAnalysis.h" |
| 16 | #include "NVPTXMachineFunctionInfo.h" |
| 17 | #include "NVPTXTargetObjectFile.h" |
| 18 | #include "NVPTXTargetTransformInfo.h" |
| 19 | #include "TargetInfo/NVPTXTargetInfo.h" |
| 20 | #include "llvm/Analysis/TargetTransformInfo.h" |
| 21 | #include "llvm/CodeGen/Passes.h" |
| 22 | #include "llvm/CodeGen/TargetPassConfig.h" |
| 23 | #include "llvm/IR/IntrinsicsNVPTX.h" |
| 24 | #include "llvm/MC/TargetRegistry.h" |
| 25 | #include "llvm/Option/LibraryOptions.h" |
| 26 | #include "llvm/Pass.h" |
| 27 | #include "llvm/Support/Compiler.h" |
| 28 | #include "llvm/Target/TargetMachine.h" |
| 29 | #include "llvm/Target/TargetOptions.h" |
| 30 | #include "llvm/TargetParser/Triple.h" |
| 31 | #include "llvm/Transforms/IPO/ExpandVariadics.h" |
| 32 | #include "llvm/Transforms/Scalar.h" |
| 33 | #include "llvm/Transforms/Scalar/GVN.h" |
| 34 | #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h" |
| 35 | #include <cassert> |
| 36 | #include <optional> |
| 37 | #include <string> |
| 38 | |
| 39 | using namespace llvm; |
| 40 | |
| 41 | extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeNVPTXTarget() { |
| 42 | // Register the target. |
| 43 | RegisterTargetMachine<NVPTXTargetMachine> X(getTheNVPTXTarget32()); |
| 44 | RegisterTargetMachine<NVPTXTargetMachine> Y(getTheNVPTXTarget64()); |
| 45 | static opt::RegisterLibraryOptions<NVPTXOptions> O; |
| 46 | |
| 47 | PassRegistry &PR = *PassRegistry::getPassRegistry(); |
| 48 | // FIXME: This pass is really intended to be invoked during IR optimization, |
| 49 | // but it's very NVPTX-specific. |
| 50 | initializeNVVMReflectLegacyPassPass(PR); |
| 51 | initializeNVVMIntrRangePass(PR); |
| 52 | initializeGenericToNVVMLegacyPassPass(PR); |
| 53 | initializeNVPTXAllocaHoistingLegacyPassPass(PR); |
| 54 | initializeNVPTXAsmPrinterPass(PR); |
| 55 | initializeNVPTXAssignValidGlobalNamesLegacyPassPass(PR); |
| 56 | initializeNVPTXAtomicLowerLegacyPassPass(PR); |
| 57 | initializeNVPTXCanonicalizeDynamicSharedLegacyPassPass(PR); |
| 58 | initializeNVPTXLowerArgsLegacyPassPass(PR); |
| 59 | initializeNVPTXPromoteParamAlignLegacyPassPass(PR); |
| 60 | initializeNVPTXMarkKernelPtrsGlobalLegacyPassPass(PR); |
| 61 | initializeNVPTXLowerAllocaLegacyPassPass(PR); |
| 62 | initializeNVPTXLowerUnreachableLegacyPassPass(PR); |
| 63 | initializeNVPTXCtorDtorLoweringLegacyPass(PR); |
| 64 | initializeNVPTXLowerAggrCopiesLegacyPassPass(PR); |
| 65 | initializeNVPTXProxyRegErasureLegacyPassPass(PR); |
| 66 | initializeNVPTXForwardParamsLegacyPassPass(PR); |
| 67 | initializeNVPTXAddressFolderLegacyPassPass(PR); |
| 68 | initializeNVPTXDAGToDAGISelLegacyPass(PR); |
| 69 | initializeNVPTXAAWrapperPassPass(PR); |
| 70 | initializeNVPTXExternalAAWrapperPass(PR); |
| 71 | initializeNVPTXPeepholeLegacyPassPass(PR); |
| 72 | initializeNVPTXTagInvariantLoadLegacyPassPass(PR); |
| 73 | initializeNVPTXIRPeepholePass(PR); |
| 74 | initializeNVPTXPrologEpilogLegacyPassPass(PR); |
| 75 | } |
| 76 | |
| 77 | NVPTXTargetMachine::NVPTXTargetMachine(const Target &T, const Triple &TT, |
| 78 | StringRef CPU, StringRef FS, |
| 79 | const TargetOptions &Options, |
| 80 | std::optional<Reloc::Model> RM, |
| 81 | std::optional<CodeModel::Model> CM, |
| 82 | CodeGenOptLevel OL, bool JIT) |
| 83 | // The pic relocation model is used regardless of what the client has |
| 84 | // specified, as it is the only relocation model currently supported. |
| 85 | : CodeGenTargetMachineImpl(T, TT, CPU, FS, Options, Reloc::PIC_, |
| 86 | getEffectiveCodeModel(CM, Default: CodeModel::Small), OL), |
| 87 | CLOpts(cl::LibraryOptionsContext::get<NVPTXOptions>( |
| 88 | C: Options.MCOptions.LibraryOptions)), |
| 89 | TLOF(std::make_unique<NVPTXTargetObjectFile>()), |
| 90 | Subtarget(TT, CPU, FS, *this) { |
| 91 | if (!CLOpts.disable_nvptx_require_structured_cfg) |
| 92 | setRequiresStructuredCFG(true); |
| 93 | // NVPTX does not produce verifier-clean MIR yet; see isMachineVerifierClean() |
| 94 | // for the legacy pass manager equivalent. |
| 95 | setEnableDefaultMachineVerifier(false); |
| 96 | initAsmInfo(); |
| 97 | } |
| 98 | |
| 99 | NVPTXTargetMachine::~NVPTXTargetMachine() = default; |
| 100 | |
| 101 | namespace { |
| 102 | |
| 103 | /// NVPTXPassConfig mirrors the NewPM implementation in NVPTXCodeGenPassBuilder |
| 104 | /// in NVPTXCodeGenPassBuilder.cpp; the two must be kept in sync until this path |
| 105 | /// is removed. |
| 106 | class NVPTXPassConfig : public TargetPassConfig { |
| 107 | const NVPTXOptions &CLOpts; |
| 108 | |
| 109 | public: |
| 110 | NVPTXPassConfig(NVPTXTargetMachine &TM, PassManagerBase &PM) |
| 111 | : TargetPassConfig(TM, PM), CLOpts(TM.getCLOpts()) {} |
| 112 | |
| 113 | NVPTXTargetMachine &getNVPTXTargetMachine() const { |
| 114 | return getTM<NVPTXTargetMachine>(); |
| 115 | } |
| 116 | |
| 117 | void addIRPasses() override; |
| 118 | bool addInstSelector() override; |
| 119 | void addPreRegAlloc() override; |
| 120 | void addPostRegAlloc() override; |
| 121 | |
| 122 | FunctionPass *createTargetRegisterAllocator(bool) override; |
| 123 | void addFastRegAlloc() override; |
| 124 | void addOptimizedRegAlloc() override; |
| 125 | |
| 126 | bool addRegAssignAndRewriteFast() override { |
| 127 | llvm_unreachable("should not be used" ); |
| 128 | } |
| 129 | |
| 130 | bool addRegAssignAndRewriteOptimized() override { |
| 131 | llvm_unreachable("should not be used" ); |
| 132 | } |
| 133 | |
| 134 | private: |
| 135 | // If the opt level is aggressive, add GVN; otherwise, add EarlyCSE. This |
| 136 | // function is only called in opt mode. |
| 137 | void addEarlyCSEOrGVNPass(); |
| 138 | |
| 139 | // Add passes that propagate special memory spaces. |
| 140 | void addAddressSpaceInferencePasses(); |
| 141 | |
| 142 | // Add passes that perform straight-line scalar optimizations. |
| 143 | void addStraightLineScalarOptimizationPasses(); |
| 144 | }; |
| 145 | |
| 146 | } // end anonymous namespace |
| 147 | |
| 148 | TargetPassConfig *NVPTXTargetMachine::createPassConfig(PassManagerBase &PM) { |
| 149 | return new NVPTXPassConfig(*this, PM); |
| 150 | } |
| 151 | |
| 152 | MachineFunctionInfo *NVPTXTargetMachine::createMachineFunctionInfo( |
| 153 | BumpPtrAllocator &Allocator, const Function &F, |
| 154 | const TargetSubtargetInfo *STI) const { |
| 155 | return NVPTXMachineFunctionInfo::create<NVPTXMachineFunctionInfo>(Allocator, |
| 156 | F, STI); |
| 157 | } |
| 158 | |
| 159 | void NVPTXTargetMachine::registerEarlyDefaultAliasAnalyses(AAManager &AAM) { |
| 160 | AAM.registerFunctionAnalysis<NVPTXAA>(); |
| 161 | } |
| 162 | |
| 163 | TargetTransformInfo |
| 164 | NVPTXTargetMachine::getTargetTransformInfo(const Function &F) const { |
| 165 | return TargetTransformInfo(std::make_unique<NVPTXTTIImpl>(args: this, args: F)); |
| 166 | } |
| 167 | |
| 168 | std::pair<const Value *, unsigned> |
| 169 | NVPTXTargetMachine::getPredicatedAddrSpace(const Value *V) const { |
| 170 | if (auto *II = dyn_cast<IntrinsicInst>(Val: V)) { |
| 171 | switch (II->getIntrinsicID()) { |
| 172 | case Intrinsic::nvvm_isspacep_const: |
| 173 | return std::make_pair(x: II->getArgOperand(i: 0), y: llvm::ADDRESS_SPACE_CONST); |
| 174 | case Intrinsic::nvvm_isspacep_global: |
| 175 | return std::make_pair(x: II->getArgOperand(i: 0), y: llvm::ADDRESS_SPACE_GLOBAL); |
| 176 | case Intrinsic::nvvm_isspacep_local: |
| 177 | return std::make_pair(x: II->getArgOperand(i: 0), y: llvm::ADDRESS_SPACE_LOCAL); |
| 178 | case Intrinsic::nvvm_isspacep_shared: |
| 179 | return std::make_pair(x: II->getArgOperand(i: 0), y: llvm::ADDRESS_SPACE_SHARED); |
| 180 | case Intrinsic::nvvm_isspacep_shared_cluster: |
| 181 | return std::make_pair(x: II->getArgOperand(i: 0), |
| 182 | y: llvm::ADDRESS_SPACE_SHARED_CLUSTER); |
| 183 | default: |
| 184 | break; |
| 185 | } |
| 186 | } |
| 187 | return std::make_pair(x: nullptr, y: -1); |
| 188 | } |
| 189 | |
| 190 | void NVPTXPassConfig::addEarlyCSEOrGVNPass() { |
| 191 | if (getOptLevel() == CodeGenOptLevel::Aggressive) |
| 192 | // Disable scalar PRE due to Register Pressure increase |
| 193 | addPass(P: createGVNPass(/*ScalarPRE=*/false)); |
| 194 | else |
| 195 | addPass(P: createEarlyCSEPass()); |
| 196 | } |
| 197 | |
| 198 | void NVPTXPassConfig::addAddressSpaceInferencePasses() { |
| 199 | // NVPTXLowerArgs emits alloca for byval parameters which can often |
| 200 | // be eliminated by SROA. |
| 201 | addPass(P: createSROAPass(/*PreserveCFG=*/true, |
| 202 | /*AggregateToVector=*/true)); |
| 203 | addPass(P: createNVPTXLowerAllocaLegacyPass()); |
| 204 | // TODO: Consider running InferAddressSpaces during opt, earlier in the |
| 205 | // compilation flow. |
| 206 | addPass(P: createInferAddressSpacesPass()); |
| 207 | addPass(P: createNVPTXAtomicLowerLegacyPass()); |
| 208 | } |
| 209 | |
| 210 | void NVPTXPassConfig::addStraightLineScalarOptimizationPasses() { |
| 211 | addPass(P: createSeparateConstOffsetFromGEPPass()); |
| 212 | addPass(P: createSpeculativeExecutionPass()); |
| 213 | // ReassociateGEPs exposes more opportunites for SLSR. See |
| 214 | // the example in reassociate-geps-and-slsr.ll. |
| 215 | addPass(P: createStraightLineStrengthReducePass()); |
| 216 | // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or |
| 217 | // EarlyCSE can reuse. GVN generates significantly better code than EarlyCSE |
| 218 | // for some of our benchmarks. |
| 219 | addEarlyCSEOrGVNPass(); |
| 220 | // Run NaryReassociate after EarlyCSE/GVN to be more effective. |
| 221 | addPass(P: createNaryReassociatePass()); |
| 222 | // NaryReassociate on GEPs creates redundant common expressions, so run |
| 223 | // EarlyCSE after it. |
| 224 | addPass(P: createEarlyCSEPass()); |
| 225 | } |
| 226 | |
| 227 | void NVPTXPassConfig::addIRPasses() { |
| 228 | // The following passes are known to not play well with virtual regs hanging |
| 229 | // around after register allocation (which in our case, is *all* registers). |
| 230 | // We explicitly disable them here. We do, however, need some functionality |
| 231 | // of the PrologEpilogCodeInserter pass, so we emulate that behavior in the |
| 232 | // NVPTXPrologEpilog pass (see NVPTXPrologEpilogPass.cpp). |
| 233 | disablePass(PassID: &PrologEpilogCodeInserterID); |
| 234 | disablePass(PassID: &MachineLateInstrsCleanupID); |
| 235 | disablePass(PassID: &MachineCopyPropagationID); |
| 236 | disablePass(PassID: &TailDuplicateLegacyID); |
| 237 | disablePass(PassID: &StackMapLivenessID); |
| 238 | disablePass(PassID: &PostRAMachineSinkingID); |
| 239 | disablePass(PassID: &PostRASchedulerID); |
| 240 | disablePass(PassID: &FuncletLayoutID); |
| 241 | disablePass(PassID: &PatchableFunctionID); |
| 242 | disablePass(PassID: &ShrinkWrapID); |
| 243 | disablePass(PassID: &RemoveLoadsIntoFakeUsesID); |
| 244 | |
| 245 | // Leave block placement to ptxas. Its CFG transformations can invalidate |
| 246 | // layout decisions made at the PTX level, and block placement here can |
| 247 | // interfere with ptxas analyses. |
| 248 | disablePass(PassID: &MachineBlockPlacementID); |
| 249 | |
| 250 | addPass(P: createNVPTXAAWrapperPass()); |
| 251 | addPass(P: createNVPTXExternalAAWrapperPass()); |
| 252 | |
| 253 | // NVVMReflectPass is added in addEarlyAsPossiblePasses, so hopefully running |
| 254 | // it here does nothing. But since we need it for correctness when lowering |
| 255 | // to NVPTX, run it here too, in case whoever built our pass pipeline didn't |
| 256 | // call addEarlyAsPossiblePasses. |
| 257 | const NVPTXSubtarget &ST = *getTM<NVPTXTargetMachine>().getSubtargetImpl(); |
| 258 | addPass(P: createNVVMReflectPass(SmVersion: ST.getSmVersion())); |
| 259 | addPass(P: createNVPTXCanonicalizeDynamicSharedLegacyPass()); |
| 260 | |
| 261 | if (getOptLevel() != CodeGenOptLevel::None) |
| 262 | addPass(P: createNVPTXImageOptimizerLegacyPass()); |
| 263 | addPass(P: createNVPTXAssignValidGlobalNamesLegacyPass()); |
| 264 | addPass(P: createGenericToNVVMLegacyPass()); |
| 265 | |
| 266 | // Lower variadic calls before address space inference. |
| 267 | addPass(P: createExpandVariadicsPass(ExpandVariadicsMode::Lowering)); |
| 268 | |
| 269 | // NVPTXLowerArgs is required for correctness and should be run right |
| 270 | // before the address space inference passes. |
| 271 | if (getNVPTXTargetMachine().getDrvInterface() == NVPTX::CUDA) |
| 272 | addPass(P: createNVPTXMarkKernelPtrsGlobalPass()); |
| 273 | addPass(P: createNVPTXPromoteParamAlignPass()); |
| 274 | addPass(P: createNVPTXLowerArgsPass()); |
| 275 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 276 | addAddressSpaceInferencePasses(); |
| 277 | addStraightLineScalarOptimizationPasses(); |
| 278 | } else { |
| 279 | // Required for correct stack lowering |
| 280 | addPass(P: createNVPTXLowerAllocaLegacyPass()); |
| 281 | } |
| 282 | |
| 283 | addPass(P: createAtomicExpandLegacyPass()); |
| 284 | addPass(P: createNVPTXCtorDtorLoweringLegacyPass()); |
| 285 | |
| 286 | // === LSR and other generic IR passes === |
| 287 | TargetPassConfig::addIRPasses(); |
| 288 | // EarlyCSE is not always strong enough to clean up what LSR produces. For |
| 289 | // example, GVN can combine |
| 290 | // |
| 291 | // %0 = add %a, %b |
| 292 | // %1 = add %b, %a |
| 293 | // |
| 294 | // and |
| 295 | // |
| 296 | // %0 = shl nsw %a, 2 |
| 297 | // %1 = shl %a, 2 |
| 298 | // |
| 299 | // but EarlyCSE can do neither of them. |
| 300 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 301 | addEarlyCSEOrGVNPass(); |
| 302 | if (!CLOpts.disable_nvptx_load_store_vectorizer) |
| 303 | addPass(P: createLoadStoreVectorizerPass()); |
| 304 | addPass(P: createSROAPass(/*PreserveCFG=*/true, |
| 305 | /*AggregateToVector=*/true)); |
| 306 | addPass(P: createNVPTXTagInvariantLoadsPass()); |
| 307 | if (!CLOpts.disable_nvptx_ir_peephole) |
| 308 | addPass(P: createNVPTXIRPeepholePass()); |
| 309 | } |
| 310 | |
| 311 | if (ST.hasPTXASUnreachableBug()) { |
| 312 | // Run LowerUnreachable to WAR a ptxas bug. See the commit description of |
| 313 | // 1ee4d880e8760256c606fe55b7af85a4f70d006d for more details. |
| 314 | const auto &Options = getNVPTXTargetMachine().Options; |
| 315 | addPass(P: createNVPTXLowerUnreachableLegacyPass(TrapUnreachable: Options.TrapUnreachable, |
| 316 | NoTrapAfterNoreturn: Options.NoTrapAfterNoreturn)); |
| 317 | } |
| 318 | } |
| 319 | |
| 320 | bool NVPTXPassConfig::addInstSelector() { |
| 321 | addPass(P: createNVPTXLowerAggrCopiesLegacyPass()); |
| 322 | addPass(P: createNVPTXAllocaHoistingLegacyPass()); |
| 323 | addPass(P: createNVPTXISelDag(TM&: getNVPTXTargetMachine(), OptLevel: getOptLevel())); |
| 324 | addPass(P: createNVPTXReplaceImageHandlesLegacyPass()); |
| 325 | |
| 326 | return false; |
| 327 | } |
| 328 | |
| 329 | void NVPTXPassConfig::addPreRegAlloc() { |
| 330 | addPass(P: createNVPTXForwardParamsLegacyPass()); |
| 331 | if (getOptLevel() != CodeGenOptLevel::None) |
| 332 | addPass(P: createNVPTXAddressFolderLegacyPass()); |
| 333 | // Remove Proxy Register pseudo instructions used to keep `callseq_end` alive. |
| 334 | addPass(P: createNVPTXProxyRegErasureLegacyPass()); |
| 335 | } |
| 336 | |
| 337 | void NVPTXPassConfig::addPostRegAlloc() { |
| 338 | addPass(P: createNVPTXPrologEpilogLegacyPass()); |
| 339 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 340 | // NVPTXPrologEpilogPass calculates frame object offset and replace frame |
| 341 | // index with VRFrame register. NVPTXPeephole need to be run after that and |
| 342 | // will replace VRFrame with VRFrameLocal when possible. |
| 343 | addPass(P: createNVPTXPeepholeLegacyPass()); |
| 344 | } |
| 345 | } |
| 346 | |
| 347 | FunctionPass *NVPTXPassConfig::createTargetRegisterAllocator(bool) { |
| 348 | return nullptr; // No reg alloc |
| 349 | } |
| 350 | |
| 351 | void NVPTXPassConfig::addFastRegAlloc() { |
| 352 | addPass(PassID: &PHIEliminationID); |
| 353 | addPass(PassID: &TwoAddressInstructionPassID); |
| 354 | } |
| 355 | |
| 356 | void NVPTXPassConfig::addOptimizedRegAlloc() { |
| 357 | addPass(PassID: &ProcessImplicitDefsID); |
| 358 | addPass(PassID: &MachineLoopInfoID); |
| 359 | addPass(PassID: &PHIEliminationID); |
| 360 | |
| 361 | addPass(PassID: &TwoAddressInstructionPassID); |
| 362 | addPass(PassID: &RegisterCoalescerID); |
| 363 | |
| 364 | // PreRA instruction scheduling. |
| 365 | if (addPass(PassID: &MachineSchedulerID)) |
| 366 | printAndVerify(Banner: "After Machine Scheduling" ); |
| 367 | |
| 368 | addPass(PassID: &StackSlotColoringID); |
| 369 | |
| 370 | // FIXME: Needs physical registers |
| 371 | // addPass(&MachineLICMID); |
| 372 | |
| 373 | printAndVerify(Banner: "After StackSlotColoring" ); |
| 374 | } |
| 375 | |