1//===-- NVPTXTargetMachine.cpp - Define TargetMachine for NVPTX -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Top-level implementation for the NVPTX target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "NVPTXTargetMachine.h"
14#include "NVPTX.h"
15#include "NVPTXAliasAnalysis.h"
16#include "NVPTXMachineFunctionInfo.h"
17#include "NVPTXTargetObjectFile.h"
18#include "NVPTXTargetTransformInfo.h"
19#include "TargetInfo/NVPTXTargetInfo.h"
20#include "llvm/Analysis/TargetTransformInfo.h"
21#include "llvm/CodeGen/Passes.h"
22#include "llvm/CodeGen/TargetPassConfig.h"
23#include "llvm/IR/IntrinsicsNVPTX.h"
24#include "llvm/MC/TargetRegistry.h"
25#include "llvm/Option/LibraryOptions.h"
26#include "llvm/Pass.h"
27#include "llvm/Support/Compiler.h"
28#include "llvm/Target/TargetMachine.h"
29#include "llvm/Target/TargetOptions.h"
30#include "llvm/TargetParser/Triple.h"
31#include "llvm/Transforms/IPO/ExpandVariadics.h"
32#include "llvm/Transforms/Scalar.h"
33#include "llvm/Transforms/Scalar/GVN.h"
34#include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
35#include <cassert>
36#include <optional>
37#include <string>
38
39using namespace llvm;
40
41extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeNVPTXTarget() {
42 // Register the target.
43 RegisterTargetMachine<NVPTXTargetMachine> X(getTheNVPTXTarget32());
44 RegisterTargetMachine<NVPTXTargetMachine> Y(getTheNVPTXTarget64());
45 static opt::RegisterLibraryOptions<NVPTXOptions> O;
46
47 PassRegistry &PR = *PassRegistry::getPassRegistry();
48 // FIXME: This pass is really intended to be invoked during IR optimization,
49 // but it's very NVPTX-specific.
50 initializeNVVMReflectLegacyPassPass(PR);
51 initializeNVVMIntrRangePass(PR);
52 initializeGenericToNVVMLegacyPassPass(PR);
53 initializeNVPTXAllocaHoistingLegacyPassPass(PR);
54 initializeNVPTXAsmPrinterPass(PR);
55 initializeNVPTXAssignValidGlobalNamesLegacyPassPass(PR);
56 initializeNVPTXAtomicLowerLegacyPassPass(PR);
57 initializeNVPTXCanonicalizeDynamicSharedLegacyPassPass(PR);
58 initializeNVPTXLowerArgsLegacyPassPass(PR);
59 initializeNVPTXPromoteParamAlignLegacyPassPass(PR);
60 initializeNVPTXMarkKernelPtrsGlobalLegacyPassPass(PR);
61 initializeNVPTXLowerAllocaLegacyPassPass(PR);
62 initializeNVPTXLowerUnreachableLegacyPassPass(PR);
63 initializeNVPTXCtorDtorLoweringLegacyPass(PR);
64 initializeNVPTXLowerAggrCopiesLegacyPassPass(PR);
65 initializeNVPTXProxyRegErasureLegacyPassPass(PR);
66 initializeNVPTXForwardParamsLegacyPassPass(PR);
67 initializeNVPTXAddressFolderLegacyPassPass(PR);
68 initializeNVPTXDAGToDAGISelLegacyPass(PR);
69 initializeNVPTXAAWrapperPassPass(PR);
70 initializeNVPTXExternalAAWrapperPass(PR);
71 initializeNVPTXPeepholeLegacyPassPass(PR);
72 initializeNVPTXTagInvariantLoadLegacyPassPass(PR);
73 initializeNVPTXIRPeepholePass(PR);
74 initializeNVPTXPrologEpilogLegacyPassPass(PR);
75}
76
77NVPTXTargetMachine::NVPTXTargetMachine(const Target &T, const Triple &TT,
78 StringRef CPU, StringRef FS,
79 const TargetOptions &Options,
80 std::optional<Reloc::Model> RM,
81 std::optional<CodeModel::Model> CM,
82 CodeGenOptLevel OL, bool JIT)
83 // The pic relocation model is used regardless of what the client has
84 // specified, as it is the only relocation model currently supported.
85 : CodeGenTargetMachineImpl(T, TT, CPU, FS, Options, Reloc::PIC_,
86 getEffectiveCodeModel(CM, Default: CodeModel::Small), OL),
87 CLOpts(cl::LibraryOptionsContext::get<NVPTXOptions>(
88 C: Options.MCOptions.LibraryOptions)),
89 TLOF(std::make_unique<NVPTXTargetObjectFile>()),
90 Subtarget(TT, CPU, FS, *this) {
91 if (!CLOpts.disable_nvptx_require_structured_cfg)
92 setRequiresStructuredCFG(true);
93 // NVPTX does not produce verifier-clean MIR yet; see isMachineVerifierClean()
94 // for the legacy pass manager equivalent.
95 setEnableDefaultMachineVerifier(false);
96 initAsmInfo();
97}
98
99NVPTXTargetMachine::~NVPTXTargetMachine() = default;
100
101namespace {
102
103/// NVPTXPassConfig mirrors the NewPM implementation in NVPTXCodeGenPassBuilder
104/// in NVPTXCodeGenPassBuilder.cpp; the two must be kept in sync until this path
105/// is removed.
106class NVPTXPassConfig : public TargetPassConfig {
107 const NVPTXOptions &CLOpts;
108
109public:
110 NVPTXPassConfig(NVPTXTargetMachine &TM, PassManagerBase &PM)
111 : TargetPassConfig(TM, PM), CLOpts(TM.getCLOpts()) {}
112
113 NVPTXTargetMachine &getNVPTXTargetMachine() const {
114 return getTM<NVPTXTargetMachine>();
115 }
116
117 void addIRPasses() override;
118 bool addInstSelector() override;
119 void addPreRegAlloc() override;
120 void addPostRegAlloc() override;
121
122 FunctionPass *createTargetRegisterAllocator(bool) override;
123 void addFastRegAlloc() override;
124 void addOptimizedRegAlloc() override;
125
126 bool addRegAssignAndRewriteFast() override {
127 llvm_unreachable("should not be used");
128 }
129
130 bool addRegAssignAndRewriteOptimized() override {
131 llvm_unreachable("should not be used");
132 }
133
134private:
135 // If the opt level is aggressive, add GVN; otherwise, add EarlyCSE. This
136 // function is only called in opt mode.
137 void addEarlyCSEOrGVNPass();
138
139 // Add passes that propagate special memory spaces.
140 void addAddressSpaceInferencePasses();
141
142 // Add passes that perform straight-line scalar optimizations.
143 void addStraightLineScalarOptimizationPasses();
144};
145
146} // end anonymous namespace
147
148TargetPassConfig *NVPTXTargetMachine::createPassConfig(PassManagerBase &PM) {
149 return new NVPTXPassConfig(*this, PM);
150}
151
152MachineFunctionInfo *NVPTXTargetMachine::createMachineFunctionInfo(
153 BumpPtrAllocator &Allocator, const Function &F,
154 const TargetSubtargetInfo *STI) const {
155 return NVPTXMachineFunctionInfo::create<NVPTXMachineFunctionInfo>(Allocator,
156 F, STI);
157}
158
159void NVPTXTargetMachine::registerEarlyDefaultAliasAnalyses(AAManager &AAM) {
160 AAM.registerFunctionAnalysis<NVPTXAA>();
161}
162
163TargetTransformInfo
164NVPTXTargetMachine::getTargetTransformInfo(const Function &F) const {
165 return TargetTransformInfo(std::make_unique<NVPTXTTIImpl>(args: this, args: F));
166}
167
168std::pair<const Value *, unsigned>
169NVPTXTargetMachine::getPredicatedAddrSpace(const Value *V) const {
170 if (auto *II = dyn_cast<IntrinsicInst>(Val: V)) {
171 switch (II->getIntrinsicID()) {
172 case Intrinsic::nvvm_isspacep_const:
173 return std::make_pair(x: II->getArgOperand(i: 0), y: llvm::ADDRESS_SPACE_CONST);
174 case Intrinsic::nvvm_isspacep_global:
175 return std::make_pair(x: II->getArgOperand(i: 0), y: llvm::ADDRESS_SPACE_GLOBAL);
176 case Intrinsic::nvvm_isspacep_local:
177 return std::make_pair(x: II->getArgOperand(i: 0), y: llvm::ADDRESS_SPACE_LOCAL);
178 case Intrinsic::nvvm_isspacep_shared:
179 return std::make_pair(x: II->getArgOperand(i: 0), y: llvm::ADDRESS_SPACE_SHARED);
180 case Intrinsic::nvvm_isspacep_shared_cluster:
181 return std::make_pair(x: II->getArgOperand(i: 0),
182 y: llvm::ADDRESS_SPACE_SHARED_CLUSTER);
183 default:
184 break;
185 }
186 }
187 return std::make_pair(x: nullptr, y: -1);
188}
189
190void NVPTXPassConfig::addEarlyCSEOrGVNPass() {
191 if (getOptLevel() == CodeGenOptLevel::Aggressive)
192 // Disable scalar PRE due to Register Pressure increase
193 addPass(P: createGVNPass(/*ScalarPRE=*/false));
194 else
195 addPass(P: createEarlyCSEPass());
196}
197
198void NVPTXPassConfig::addAddressSpaceInferencePasses() {
199 // NVPTXLowerArgs emits alloca for byval parameters which can often
200 // be eliminated by SROA.
201 addPass(P: createSROAPass(/*PreserveCFG=*/true,
202 /*AggregateToVector=*/true));
203 addPass(P: createNVPTXLowerAllocaLegacyPass());
204 // TODO: Consider running InferAddressSpaces during opt, earlier in the
205 // compilation flow.
206 addPass(P: createInferAddressSpacesPass());
207 addPass(P: createNVPTXAtomicLowerLegacyPass());
208}
209
210void NVPTXPassConfig::addStraightLineScalarOptimizationPasses() {
211 addPass(P: createSeparateConstOffsetFromGEPPass());
212 addPass(P: createSpeculativeExecutionPass());
213 // ReassociateGEPs exposes more opportunites for SLSR. See
214 // the example in reassociate-geps-and-slsr.ll.
215 addPass(P: createStraightLineStrengthReducePass());
216 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or
217 // EarlyCSE can reuse. GVN generates significantly better code than EarlyCSE
218 // for some of our benchmarks.
219 addEarlyCSEOrGVNPass();
220 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
221 addPass(P: createNaryReassociatePass());
222 // NaryReassociate on GEPs creates redundant common expressions, so run
223 // EarlyCSE after it.
224 addPass(P: createEarlyCSEPass());
225}
226
227void NVPTXPassConfig::addIRPasses() {
228 // The following passes are known to not play well with virtual regs hanging
229 // around after register allocation (which in our case, is *all* registers).
230 // We explicitly disable them here. We do, however, need some functionality
231 // of the PrologEpilogCodeInserter pass, so we emulate that behavior in the
232 // NVPTXPrologEpilog pass (see NVPTXPrologEpilogPass.cpp).
233 disablePass(PassID: &PrologEpilogCodeInserterID);
234 disablePass(PassID: &MachineLateInstrsCleanupID);
235 disablePass(PassID: &MachineCopyPropagationID);
236 disablePass(PassID: &TailDuplicateLegacyID);
237 disablePass(PassID: &StackMapLivenessID);
238 disablePass(PassID: &PostRAMachineSinkingID);
239 disablePass(PassID: &PostRASchedulerID);
240 disablePass(PassID: &FuncletLayoutID);
241 disablePass(PassID: &PatchableFunctionID);
242 disablePass(PassID: &ShrinkWrapID);
243 disablePass(PassID: &RemoveLoadsIntoFakeUsesID);
244
245 // Leave block placement to ptxas. Its CFG transformations can invalidate
246 // layout decisions made at the PTX level, and block placement here can
247 // interfere with ptxas analyses.
248 disablePass(PassID: &MachineBlockPlacementID);
249
250 addPass(P: createNVPTXAAWrapperPass());
251 addPass(P: createNVPTXExternalAAWrapperPass());
252
253 // NVVMReflectPass is added in addEarlyAsPossiblePasses, so hopefully running
254 // it here does nothing. But since we need it for correctness when lowering
255 // to NVPTX, run it here too, in case whoever built our pass pipeline didn't
256 // call addEarlyAsPossiblePasses.
257 const NVPTXSubtarget &ST = *getTM<NVPTXTargetMachine>().getSubtargetImpl();
258 addPass(P: createNVVMReflectPass(SmVersion: ST.getSmVersion()));
259 addPass(P: createNVPTXCanonicalizeDynamicSharedLegacyPass());
260
261 if (getOptLevel() != CodeGenOptLevel::None)
262 addPass(P: createNVPTXImageOptimizerLegacyPass());
263 addPass(P: createNVPTXAssignValidGlobalNamesLegacyPass());
264 addPass(P: createGenericToNVVMLegacyPass());
265
266 // Lower variadic calls before address space inference.
267 addPass(P: createExpandVariadicsPass(ExpandVariadicsMode::Lowering));
268
269 // NVPTXLowerArgs is required for correctness and should be run right
270 // before the address space inference passes.
271 if (getNVPTXTargetMachine().getDrvInterface() == NVPTX::CUDA)
272 addPass(P: createNVPTXMarkKernelPtrsGlobalPass());
273 addPass(P: createNVPTXPromoteParamAlignPass());
274 addPass(P: createNVPTXLowerArgsPass());
275 if (getOptLevel() != CodeGenOptLevel::None) {
276 addAddressSpaceInferencePasses();
277 addStraightLineScalarOptimizationPasses();
278 } else {
279 // Required for correct stack lowering
280 addPass(P: createNVPTXLowerAllocaLegacyPass());
281 }
282
283 addPass(P: createAtomicExpandLegacyPass());
284 addPass(P: createNVPTXCtorDtorLoweringLegacyPass());
285
286 // === LSR and other generic IR passes ===
287 TargetPassConfig::addIRPasses();
288 // EarlyCSE is not always strong enough to clean up what LSR produces. For
289 // example, GVN can combine
290 //
291 // %0 = add %a, %b
292 // %1 = add %b, %a
293 //
294 // and
295 //
296 // %0 = shl nsw %a, 2
297 // %1 = shl %a, 2
298 //
299 // but EarlyCSE can do neither of them.
300 if (getOptLevel() != CodeGenOptLevel::None) {
301 addEarlyCSEOrGVNPass();
302 if (!CLOpts.disable_nvptx_load_store_vectorizer)
303 addPass(P: createLoadStoreVectorizerPass());
304 addPass(P: createSROAPass(/*PreserveCFG=*/true,
305 /*AggregateToVector=*/true));
306 addPass(P: createNVPTXTagInvariantLoadsPass());
307 if (!CLOpts.disable_nvptx_ir_peephole)
308 addPass(P: createNVPTXIRPeepholePass());
309 }
310
311 if (ST.hasPTXASUnreachableBug()) {
312 // Run LowerUnreachable to WAR a ptxas bug. See the commit description of
313 // 1ee4d880e8760256c606fe55b7af85a4f70d006d for more details.
314 const auto &Options = getNVPTXTargetMachine().Options;
315 addPass(P: createNVPTXLowerUnreachableLegacyPass(TrapUnreachable: Options.TrapUnreachable,
316 NoTrapAfterNoreturn: Options.NoTrapAfterNoreturn));
317 }
318}
319
320bool NVPTXPassConfig::addInstSelector() {
321 addPass(P: createNVPTXLowerAggrCopiesLegacyPass());
322 addPass(P: createNVPTXAllocaHoistingLegacyPass());
323 addPass(P: createNVPTXISelDag(TM&: getNVPTXTargetMachine(), OptLevel: getOptLevel()));
324 addPass(P: createNVPTXReplaceImageHandlesLegacyPass());
325
326 return false;
327}
328
329void NVPTXPassConfig::addPreRegAlloc() {
330 addPass(P: createNVPTXForwardParamsLegacyPass());
331 if (getOptLevel() != CodeGenOptLevel::None)
332 addPass(P: createNVPTXAddressFolderLegacyPass());
333 // Remove Proxy Register pseudo instructions used to keep `callseq_end` alive.
334 addPass(P: createNVPTXProxyRegErasureLegacyPass());
335}
336
337void NVPTXPassConfig::addPostRegAlloc() {
338 addPass(P: createNVPTXPrologEpilogLegacyPass());
339 if (getOptLevel() != CodeGenOptLevel::None) {
340 // NVPTXPrologEpilogPass calculates frame object offset and replace frame
341 // index with VRFrame register. NVPTXPeephole need to be run after that and
342 // will replace VRFrame with VRFrameLocal when possible.
343 addPass(P: createNVPTXPeepholeLegacyPass());
344 }
345}
346
347FunctionPass *NVPTXPassConfig::createTargetRegisterAllocator(bool) {
348 return nullptr; // No reg alloc
349}
350
351void NVPTXPassConfig::addFastRegAlloc() {
352 addPass(PassID: &PHIEliminationID);
353 addPass(PassID: &TwoAddressInstructionPassID);
354}
355
356void NVPTXPassConfig::addOptimizedRegAlloc() {
357 addPass(PassID: &ProcessImplicitDefsID);
358 addPass(PassID: &MachineLoopInfoID);
359 addPass(PassID: &PHIEliminationID);
360
361 addPass(PassID: &TwoAddressInstructionPassID);
362 addPass(PassID: &RegisterCoalescerID);
363
364 // PreRA instruction scheduling.
365 if (addPass(PassID: &MachineSchedulerID))
366 printAndVerify(Banner: "After Machine Scheduling");
367
368 addPass(PassID: &StackSlotColoringID);
369
370 // FIXME: Needs physical registers
371 // addPass(&MachineLICMID);
372
373 printAndVerify(Banner: "After StackSlotColoring");
374}
375