1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file contains the NVPTX CodeGen pipeline builder. It mirrors
10/// NVPTXPassConfig in NVPTXTargetMachine.cpp; the two must be kept in sync
11/// until the legacy pass manager path is removed.
12//===----------------------------------------------------------------------===//
13
14#include "NVPTX.h"
15#include "NVPTXAliasAnalysis.h"
16#include "NVPTXAsmPrinter.h"
17#include "NVPTXSubtarget.h"
18#include "NVPTXTargetMachine.h"
19#include "llvm/Analysis/KernelInfo.h"
20#include "llvm/CodeGen/AtomicExpand.h"
21#include "llvm/CodeGen/FuncletLayout.h"
22#include "llvm/CodeGen/MachineBlockPlacement.h"
23#include "llvm/CodeGen/MachineCopyPropagation.h"
24#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
25#include "llvm/CodeGen/MachineLoopInfo.h"
26#include "llvm/CodeGen/MachineScheduler.h"
27#include "llvm/CodeGen/PEI.h"
28#include "llvm/CodeGen/PHIElimination.h"
29#include "llvm/CodeGen/PatchableFunction.h"
30#include "llvm/CodeGen/PostRAMachineSink.h"
31#include "llvm/CodeGen/PostRASchedulerList.h"
32#include "llvm/CodeGen/ProcessImplicitDefs.h"
33#include "llvm/CodeGen/RegisterCoalescerPass.h"
34#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
35#include "llvm/CodeGen/ShrinkWrap.h"
36#include "llvm/CodeGen/StackSlotColoring.h"
37#include "llvm/CodeGen/TailDuplication.h"
38#include "llvm/CodeGen/TwoAddressInstructionPass.h"
39#include "llvm/IR/PassInstrumentation.h"
40#include "llvm/MC/MCStreamer.h"
41#include "llvm/Passes/CodeGenPassBuilder.h"
42#include "llvm/Passes/PassBuilder.h"
43#include "llvm/Target/CGPassBuilderOption.h"
44#include "llvm/Transforms/IPO/ExpandVariadics.h"
45#include "llvm/Transforms/Scalar/EarlyCSE.h"
46#include "llvm/Transforms/Scalar/GVN.h"
47#include "llvm/Transforms/Scalar/InferAddressSpaces.h"
48#include "llvm/Transforms/Scalar/NaryReassociate.h"
49#include "llvm/Transforms/Scalar/SROA.h"
50#include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"
51#include "llvm/Transforms/Scalar/SpeculativeExecution.h"
52#include "llvm/Transforms/Scalar/StraightLineStrengthReduce.h"
53#include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
54
55using namespace llvm;
56
57namespace {
58
59class NVPTXCodeGenPassBuilder : public CodeGenPassBuilder {
60 using Base = CodeGenPassBuilder;
61
62 const NVPTXOptions &CLOpts;
63
64 NVPTXTargetMachine &getTM() const {
65 return static_cast<NVPTXTargetMachine &>(TM);
66 }
67
68public:
69 explicit NVPTXCodeGenPassBuilder(NVPTXTargetMachine &TM,
70 const CGPassBuilderOption &Opts,
71 PassInstrumentationCallbacks *PIC)
72 : CodeGenPassBuilder(TM, Opts, PIC), CLOpts(TM.getCLOpts()) {
73 // The following passes are known to not play well with virtual regs
74 // hanging around after register allocation (which in our case, is *all*
75 // registers). We explicitly disable them here. We do, however, need some
76 // functionality of the PrologEpilogCodeInserter pass, so we emulate that
77 // behavior in the NVPTXPrologEpilog pass (see NVPTXPrologEpilogPass.cpp).
78 disablePass<PrologEpilogInserterPass, MachineLateInstrsCleanupPass,
79 MachineCopyPropagationPass, TailDuplicatePass,
80 StackMapLivenessPass, PostRAMachineSinkingPass,
81 PostRASchedulerPass, FuncletLayoutPass, PatchableFunctionPass,
82 ShrinkWrapPass, RemoveLoadsIntoFakeUsesPass,
83 MachineBlockPlacementPass>();
84 }
85
86 void addIRPasses(PassManagerWrapper &PMW) override;
87 Error addInstSelector(PassManagerWrapper &PMW) override;
88 void addPreRegAlloc(PassManagerWrapper &PMW) override;
89 void addPostRegAlloc(PassManagerWrapper &PMW) override;
90
91 // NVPTX has no register allocation; virtual registers are emitted directly.
92 void addTargetRegisterAllocator(PassManagerWrapper &PMW, bool) override {}
93 Error addFastRegAlloc(PassManagerWrapper &PMW) override;
94 Error addOptimizedRegAlloc(PassManagerWrapper &PMW) override;
95
96 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
97 void addAsmPrinter(PassManagerWrapper &PMW) override;
98 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
99
100private:
101 // If the opt level is aggressive, add GVN; otherwise, add EarlyCSE.
102 void addEarlyCSEOrGVNPass(PassManagerWrapper &PMW);
103
104 // Add passes that propagate special memory spaces.
105 void addAddressSpaceInferencePasses(PassManagerWrapper &PMW);
106
107 // Add passes that perform straight-line scalar optimizations.
108 void addStraightLineScalarOptimizationPasses(PassManagerWrapper &PMW);
109};
110
111void NVPTXCodeGenPassBuilder::addEarlyCSEOrGVNPass(PassManagerWrapper &PMW) {
112 if (getOptLevel() == CodeGenOptLevel::Aggressive)
113 // Disable scalar PRE due to Register Pressure increase
114 addFunctionPass(Pass: GVNPass(GVNOptions().setScalarPRE(false)), PMW);
115 else
116 addFunctionPass(Pass: EarlyCSEPass(), PMW);
117}
118
119void NVPTXCodeGenPassBuilder::addAddressSpaceInferencePasses(
120 PassManagerWrapper &PMW) {
121 // NVPTXLowerArgs emits alloca for byval parameters which can often
122 // be eliminated by SROA.
123 addFunctionPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
124 /*AggregateToVector=*/true)),
125 PMW);
126 addFunctionPass(Pass: NVPTXLowerAllocaPass(), PMW);
127 // TODO: Consider running InferAddressSpaces during opt, earlier in the
128 // compilation flow.
129 addFunctionPass(Pass: InferAddressSpacesPass(), PMW);
130 addFunctionPass(Pass: NVPTXAtomicLowerPass(), PMW);
131}
132
133void NVPTXCodeGenPassBuilder::addStraightLineScalarOptimizationPasses(
134 PassManagerWrapper &PMW) {
135 addFunctionPass(Pass: SeparateConstOffsetFromGEPPass(), PMW);
136 addFunctionPass(Pass: SpeculativeExecutionPass(), PMW);
137 // ReassociateGEPs exposes more opportunites for SLSR. See
138 // the example in reassociate-geps-and-slsr.ll.
139 addFunctionPass(Pass: StraightLineStrengthReducePass(), PMW);
140 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN
141 // or EarlyCSE can reuse. GVN generates significantly better code than
142 // EarlyCSE for some of our benchmarks.
143 addEarlyCSEOrGVNPass(PMW);
144 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
145 addFunctionPass(Pass: NaryReassociatePass(), PMW);
146 // NaryReassociate on GEPs creates redundant common expressions, so run
147 // EarlyCSE after it.
148 addFunctionPass(Pass: EarlyCSEPass(), PMW);
149}
150
151void NVPTXCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
152 const NVPTXSubtarget &ST = *getTM().getSubtargetImpl();
153
154 // NVVMReflectPass is added in the pipeline-start extension point, so
155 // hopefully running it here does nothing. But since we need it for
156 // correctness when lowering to NVPTX, run it here too, in case whoever built
157 // our pass pipeline didn't add it.
158 flushFPMsToMPM(PMW);
159 addModulePass(Pass: NVVMReflectPass(ST.getSmVersion()), PMW);
160 addModulePass(Pass: NVPTXCanonicalizeDynamicSharedPass(), PMW);
161
162 if (getOptLevel() != CodeGenOptLevel::None)
163 addFunctionPass(Pass: NVPTXImageOptimizerPass(), PMW);
164 flushFPMsToMPM(PMW);
165 addModulePass(Pass: NVPTXAssignValidGlobalNamesPass(), PMW);
166 addModulePass(Pass: GenericToNVVMPass(), PMW);
167
168 // Lower variadic calls before address space inference.
169 addModulePass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Lowering), PMW);
170
171 // NVPTXLowerArgs is required for correctness and should be run right
172 // before the address space inference passes.
173 if (getTM().getDrvInterface() == NVPTX::CUDA) {
174 addFunctionPass(Pass: NVPTXMarkKernelPtrsGlobalPass(), PMW);
175 flushFPMsToMPM(PMW);
176 }
177 addModulePass(Pass: NVPTXPromoteParamAlignPass(), PMW);
178 addModulePass(Pass: NVPTXLowerArgsPass(TM), PMW);
179 if (getOptLevel() != CodeGenOptLevel::None) {
180 addAddressSpaceInferencePasses(PMW);
181 addStraightLineScalarOptimizationPasses(PMW);
182 } else {
183 // Required for correct stack lowering
184 addFunctionPass(Pass: NVPTXLowerAllocaPass(), PMW);
185 }
186
187 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
188 flushFPMsToMPM(PMW);
189 addModulePass(Pass: NVPTXCtorDtorLoweringPass(), PMW);
190
191 // === LSR and other generic IR passes ===
192 Base::addIRPasses(PMW);
193 // EarlyCSE is not always strong enough to clean up what LSR produces. For
194 // example, GVN can combine
195 //
196 // %0 = add %a, %b
197 // %1 = add %b, %a
198 //
199 // and
200 //
201 // %0 = shl nsw %a, 2
202 // %1 = shl %a, 2
203 //
204 // but EarlyCSE can do neither of them.
205 if (getOptLevel() != CodeGenOptLevel::None) {
206 addEarlyCSEOrGVNPass(PMW);
207 if (!CLOpts.disable_nvptx_load_store_vectorizer)
208 addFunctionPass(Pass: LoadStoreVectorizerPass(), PMW);
209 addFunctionPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
210 /*AggregateToVector=*/true)),
211 PMW);
212 addFunctionPass(Pass: NVPTXTagInvariantLoadsPass(), PMW);
213 if (!CLOpts.disable_nvptx_ir_peephole)
214 addFunctionPass(Pass: NVPTXIRPeepholePass(), PMW);
215 }
216
217 if (ST.hasPTXASUnreachableBug()) {
218 // Run LowerUnreachable to WAR a ptxas bug. See the commit description of
219 // 1ee4d880e8760256c606fe55b7af85a4f70d006d for more details.
220 addFunctionPass(Pass: NVPTXLowerUnreachablePass(TM.Options.TrapUnreachable,
221 TM.Options.NoTrapAfterNoreturn),
222 PMW);
223 }
224}
225
226Error NVPTXCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
227 addFunctionPass(Pass: NVPTXLowerAggrCopiesPass(), PMW);
228 addFunctionPass(Pass: NVPTXAllocaHoistingPass(), PMW);
229 addMachineFunctionPass(Pass: NVPTXISelDAGToDAGPass(getTM(), getOptLevel()), PMW);
230 addMachineFunctionPass(Pass: NVPTXReplaceImageHandlesPass(), PMW);
231 return Error::success();
232}
233
234void NVPTXCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
235 addMachineFunctionPass(Pass: NVPTXForwardParamsPass(), PMW);
236 if (getOptLevel() != CodeGenOptLevel::None)
237 addMachineFunctionPass(Pass: NVPTXAddressFolderPass(), PMW);
238 // Remove Proxy Register pseudo instructions used to keep `callseq_end` alive.
239 addMachineFunctionPass(Pass: NVPTXProxyRegErasurePass(), PMW);
240}
241
242void NVPTXCodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) {
243 addMachineFunctionPass(Pass: NVPTXPrologEpilogPass(), PMW);
244 if (getOptLevel() != CodeGenOptLevel::None) {
245 // NVPTXPrologEpilogPass calculates frame object offset and replaces frame
246 // index with VRFrame register. NVPTXPeephole needs to be run after that
247 // and will replace VRFrame with VRFrameLocal when possible.
248 addMachineFunctionPass(Pass: NVPTXPeepholePass(), PMW);
249 }
250}
251
252Error NVPTXCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
253 addMachineFunctionPass(Pass: PHIEliminationPass(), PMW);
254 addMachineFunctionPass(Pass: TwoAddressInstructionPass(), PMW);
255 return Error::success();
256}
257
258Error NVPTXCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
259 addMachineFunctionPass(Pass: ProcessImplicitDefsPass(), PMW);
260 addMachineFunctionPass(
261 Pass: RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>(), PMW);
262 addMachineFunctionPass(Pass: PHIEliminationPass(), PMW);
263
264 addMachineFunctionPass(Pass: TwoAddressInstructionPass(), PMW);
265 addMachineFunctionPass(Pass: RegisterCoalescerPass(), PMW);
266
267 // PreRA instruction scheduling.
268 addMachineFunctionPass(Pass: MachineSchedulerPass(&TM), PMW);
269
270 addMachineFunctionPass(Pass: StackSlotColoringPass(), PMW);
271
272 // FIXME: Needs physical registers
273 // addMachineFunctionPass(MachineLICMPass(), PMW);
274
275 return Error::success();
276}
277
278void NVPTXCodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
279 addModulePass(Pass: NVPTXAsmPrinterBeginPass(), PMW, /*Force=*/true);
280}
281
282void NVPTXCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
283 addMachineFunctionPass(Pass: NVPTXAsmPrinterPass(), PMW);
284}
285
286void NVPTXCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
287 addModulePass(Pass: NVPTXAsmPrinterEndPass(), PMW);
288}
289
290} // namespace
291
292void NVPTXTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
293#define GET_PASS_REGISTRY "NVPTXPassRegistry.def"
294#include "llvm/Passes/TargetPassRegistry.inc"
295
296 PB.registerPipelineStartEPCallback(
297 C: [this](ModulePassManager &PM, OptimizationLevel Level) {
298 // Before anything relies on alias analysis.
299 PM.addPass(Pass: NVPTXCanonicalizeDynamicSharedPass());
300
301 // We do not want to fold out calls to nvvm.reflect early if the user
302 // has not provided a target architecture just yet.
303 if (Subtarget.hasTargetName())
304 PM.addPass(Pass: NVVMReflectPass(Subtarget.getSmVersion()));
305
306 FunctionPassManager FPM;
307 // Note: NVVMIntrRangePass was causing numerical discrepancies at one
308 // point, if issues crop up, consider disabling.
309 FPM.addPass(Pass: NVVMIntrRangePass());
310 if (CLOpts.early_byval_copy)
311 FPM.addPass(Pass: NVPTXCopyByValArgsPass());
312 PM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
313 });
314
315 // Linking can bring in further declarations of the dynamic shared memory.
316 auto CanonicalizeAfterLink = [](ModulePassManager &PM, OptimizationLevel) {
317 PM.addPass(Pass: NVPTXCanonicalizeDynamicSharedPass());
318 };
319 PB.registerFullLinkTimeOptimizationEarlyEPCallback(C: CanonicalizeAfterLink);
320 PB.registerThinLinkTimeOptimizationEarlyEPCallback(C: CanonicalizeAfterLink);
321
322 if (!NoKernelInfoEndLTO) {
323 PB.registerFullLinkTimeOptimizationLastEPCallback(
324 C: [this](ModulePassManager &PM, OptimizationLevel Level) {
325 FunctionPassManager FPM;
326 FPM.addPass(Pass: KernelInfoPrinter(this));
327 PM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
328 });
329 }
330}
331
332Error NVPTXTargetMachine::buildCodeGenPipeline(
333 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
334 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
335 const CGPassBuilderOption &Opt, MCContext &Ctx,
336 PassInstrumentationCallbacks *PIC) {
337 auto CGPB = NVPTXCodeGenPassBuilder(*this, Opt, PIC);
338 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
339}
340