1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file contains the NVPTX CodeGen pipeline builder. It mirrors
10/// NVPTXPassConfig in NVPTXTargetMachine.cpp; the two must be kept in sync
11/// until the legacy pass manager path is removed.
12//===----------------------------------------------------------------------===//
13
14#include "NVPTX.h"
15#include "NVPTXAliasAnalysis.h"
16#include "NVPTXAsmPrinter.h"
17#include "NVPTXSubtarget.h"
18#include "NVPTXTargetMachine.h"
19#include "llvm/Analysis/KernelInfo.h"
20#include "llvm/CodeGen/AtomicExpand.h"
21#include "llvm/CodeGen/FuncletLayout.h"
22#include "llvm/CodeGen/MachineCopyPropagation.h"
23#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
24#include "llvm/CodeGen/MachineLoopInfo.h"
25#include "llvm/CodeGen/MachineScheduler.h"
26#include "llvm/CodeGen/PEI.h"
27#include "llvm/CodeGen/PHIElimination.h"
28#include "llvm/CodeGen/PatchableFunction.h"
29#include "llvm/CodeGen/PostRAMachineSink.h"
30#include "llvm/CodeGen/PostRASchedulerList.h"
31#include "llvm/CodeGen/ProcessImplicitDefs.h"
32#include "llvm/CodeGen/RegisterCoalescerPass.h"
33#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
34#include "llvm/CodeGen/ShrinkWrap.h"
35#include "llvm/CodeGen/StackSlotColoring.h"
36#include "llvm/CodeGen/TailDuplication.h"
37#include "llvm/CodeGen/TwoAddressInstructionPass.h"
38#include "llvm/CodeGen/UnreachableBlockElim.h"
39#include "llvm/IR/PassInstrumentation.h"
40#include "llvm/MC/MCStreamer.h"
41#include "llvm/Passes/CodeGenPassBuilder.h"
42#include "llvm/Passes/PassBuilder.h"
43#include "llvm/Target/CGPassBuilderOption.h"
44#include "llvm/Transforms/IPO/ExpandVariadics.h"
45#include "llvm/Transforms/Scalar/EarlyCSE.h"
46#include "llvm/Transforms/Scalar/GVN.h"
47#include "llvm/Transforms/Scalar/InferAddressSpaces.h"
48#include "llvm/Transforms/Scalar/NaryReassociate.h"
49#include "llvm/Transforms/Scalar/SROA.h"
50#include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"
51#include "llvm/Transforms/Scalar/SpeculativeExecution.h"
52#include "llvm/Transforms/Scalar/StraightLineStrengthReduce.h"
53#include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
54
55using namespace llvm;
56
57extern cl::opt<bool> DisableLoadStoreVectorizer;
58extern cl::opt<bool> DisableNVPTXIRPeephole;
59
60// byval arguments in NVPTX are special. We're only allowed to read from them
61// using a special instruction, and if we ever need to write to them or take an
62// address, we must make a local copy and use it, instead.
63//
64// The problem is that local copies are very expensive, and we create them very
65// late in the compilation pipeline, so LLVM does not have much of a chance to
66// eliminate them, if they turn out to be unnecessary.
67//
68// One way around that is to create such copies early on, and let them percolate
69// through the optimizations. The copying itself will never trigger creation of
70// another copy later on, as the reads are allowed. If LLVM can eliminate it,
71// it's a win. It the full optimization pipeline can't remove the copy, that's
72// as good as it gets in terms of the effort we could've done, and it's
73// certainly a much better effort than what we do now.
74//
75// This early injection of the copies has potential to create undesireable
76// side-effects, so it's disabled by default, for now, until it sees more
77// testing.
78static cl::opt<bool> EarlyByValArgsCopy(
79 "nvptx-early-byval-copy",
80 cl::desc("Create a copy of byval function arguments early."),
81 cl::init(Val: false), cl::Hidden);
82
83namespace {
84
85class NVPTXCodeGenPassBuilder : public CodeGenPassBuilder {
86 using Base = CodeGenPassBuilder;
87
88 NVPTXTargetMachine &getTM() const {
89 return static_cast<NVPTXTargetMachine &>(TM);
90 }
91
92public:
93 explicit NVPTXCodeGenPassBuilder(NVPTXTargetMachine &TM,
94 const CGPassBuilderOption &Opts,
95 PassInstrumentationCallbacks *PIC)
96 : CodeGenPassBuilder(TM, Opts, PIC) {
97 // The following passes are known to not play well with virtual regs
98 // hanging around after register allocation (which in our case, is *all*
99 // registers). We explicitly disable them here. We do, however, need some
100 // functionality of the PrologEpilogCodeInserter pass, so we emulate that
101 // behavior in the NVPTXPrologEpilog pass (see NVPTXPrologEpilogPass.cpp).
102 disablePass<PrologEpilogInserterPass, MachineLateInstrsCleanupPass,
103 MachineCopyPropagationPass, TailDuplicatePass,
104 StackMapLivenessPass, PostRAMachineSinkingPass,
105 PostRASchedulerPass, FuncletLayoutPass, PatchableFunctionPass,
106 ShrinkWrapPass, RemoveLoadsIntoFakeUsesPass>();
107 }
108
109 void addIRPasses(PassManagerWrapper &PMW) override;
110 Error addInstSelector(PassManagerWrapper &PMW) override;
111 void addPreRegAlloc(PassManagerWrapper &PMW) override;
112 void addPostRegAlloc(PassManagerWrapper &PMW) override;
113
114 // NVPTX has no register allocation; virtual registers are emitted directly.
115 void addTargetRegisterAllocator(PassManagerWrapper &PMW, bool) override {}
116 Error addFastRegAlloc(PassManagerWrapper &PMW) override;
117 Error addOptimizedRegAlloc(PassManagerWrapper &PMW) override;
118
119 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
120 void addAsmPrinter(PassManagerWrapper &PMW) override;
121 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
122
123private:
124 // If the opt level is aggressive, add GVN; otherwise, add EarlyCSE.
125 void addEarlyCSEOrGVNPass(PassManagerWrapper &PMW);
126
127 // Add passes that propagate special memory spaces.
128 void addAddressSpaceInferencePasses(PassManagerWrapper &PMW);
129
130 // Add passes that perform straight-line scalar optimizations.
131 void addStraightLineScalarOptimizationPasses(PassManagerWrapper &PMW);
132};
133
134void NVPTXCodeGenPassBuilder::addEarlyCSEOrGVNPass(PassManagerWrapper &PMW) {
135 if (getOptLevel() == CodeGenOptLevel::Aggressive)
136 // Disable scalar PRE due to Register Pressure increase
137 addFunctionPass(Pass: GVNPass(GVNOptions().setScalarPRE(false)), PMW);
138 else
139 addFunctionPass(Pass: EarlyCSEPass(), PMW);
140}
141
142void NVPTXCodeGenPassBuilder::addAddressSpaceInferencePasses(
143 PassManagerWrapper &PMW) {
144 // NVPTXLowerArgs emits alloca for byval parameters which can often
145 // be eliminated by SROA.
146 addFunctionPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
147 /*AggregateToVector=*/true)),
148 PMW);
149 addFunctionPass(Pass: NVPTXLowerAllocaPass(), PMW);
150 // TODO: Consider running InferAddressSpaces during opt, earlier in the
151 // compilation flow.
152 addFunctionPass(Pass: InferAddressSpacesPass(), PMW);
153 addFunctionPass(Pass: NVPTXAtomicLowerPass(), PMW);
154}
155
156void NVPTXCodeGenPassBuilder::addStraightLineScalarOptimizationPasses(
157 PassManagerWrapper &PMW) {
158 addFunctionPass(Pass: SeparateConstOffsetFromGEPPass(), PMW);
159 addFunctionPass(Pass: SpeculativeExecutionPass(), PMW);
160 // ReassociateGEPs exposes more opportunites for SLSR. See
161 // the example in reassociate-geps-and-slsr.ll.
162 addFunctionPass(Pass: StraightLineStrengthReducePass(), PMW);
163 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN
164 // or EarlyCSE can reuse. GVN generates significantly better code than
165 // EarlyCSE for some of our benchmarks.
166 addEarlyCSEOrGVNPass(PMW);
167 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
168 addFunctionPass(Pass: NaryReassociatePass(), PMW);
169 // NaryReassociate on GEPs creates redundant common expressions, so run
170 // EarlyCSE after it.
171 addFunctionPass(Pass: EarlyCSEPass(), PMW);
172}
173
174void NVPTXCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
175 const NVPTXSubtarget &ST = *getTM().getSubtargetImpl();
176
177 // NVVMReflectPass is added in the pipeline-start extension point, so
178 // hopefully running it here does nothing. But since we need it for
179 // correctness when lowering to NVPTX, run it here too, in case whoever built
180 // our pass pipeline didn't add it.
181 flushFPMsToMPM(PMW);
182 addModulePass(Pass: NVVMReflectPass(ST.getSmVersion()), PMW);
183
184 if (getOptLevel() != CodeGenOptLevel::None)
185 addFunctionPass(Pass: NVPTXImageOptimizerPass(), PMW);
186 flushFPMsToMPM(PMW);
187 addModulePass(Pass: NVPTXAssignValidGlobalNamesPass(), PMW);
188 addModulePass(Pass: GenericToNVVMPass(), PMW);
189
190 // Lower variadic calls before address space inference.
191 addModulePass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Lowering), PMW);
192
193 // NVPTXLowerArgs is required for correctness and should be run right
194 // before the address space inference passes.
195 if (getTM().getDrvInterface() == NVPTX::CUDA) {
196 addFunctionPass(Pass: NVPTXMarkKernelPtrsGlobalPass(), PMW);
197 flushFPMsToMPM(PMW);
198 }
199 addModulePass(Pass: NVPTXPromoteParamAlignPass(), PMW);
200 addModulePass(Pass: NVPTXLowerArgsPass(TM), PMW);
201 if (getOptLevel() != CodeGenOptLevel::None) {
202 addAddressSpaceInferencePasses(PMW);
203 addStraightLineScalarOptimizationPasses(PMW);
204 } else {
205 // Required for correct stack lowering
206 addFunctionPass(Pass: NVPTXLowerAllocaPass(), PMW);
207 }
208
209 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
210 flushFPMsToMPM(PMW);
211 addModulePass(Pass: NVPTXCtorDtorLoweringPass(), PMW);
212
213 // === LSR and other generic IR passes ===
214 Base::addIRPasses(PMW);
215 // EarlyCSE is not always strong enough to clean up what LSR produces. For
216 // example, GVN can combine
217 //
218 // %0 = add %a, %b
219 // %1 = add %b, %a
220 //
221 // and
222 //
223 // %0 = shl nsw %a, 2
224 // %1 = shl %a, 2
225 //
226 // but EarlyCSE can do neither of them.
227 if (getOptLevel() != CodeGenOptLevel::None) {
228 addEarlyCSEOrGVNPass(PMW);
229 if (!DisableLoadStoreVectorizer)
230 addFunctionPass(Pass: LoadStoreVectorizerPass(), PMW);
231 addFunctionPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
232 /*AggregateToVector=*/true)),
233 PMW);
234 addFunctionPass(Pass: NVPTXTagInvariantLoadsPass(), PMW);
235 if (!DisableNVPTXIRPeephole)
236 addFunctionPass(Pass: NVPTXIRPeepholePass(), PMW);
237 }
238
239 if (ST.hasPTXASUnreachableBug()) {
240 // Run LowerUnreachable to WAR a ptxas bug. See the commit description of
241 // 1ee4d880e8760256c606fe55b7af85a4f70d006d for more details.
242 addFunctionPass(Pass: NVPTXLowerUnreachablePass(TM.Options.TrapUnreachable,
243 TM.Options.NoTrapAfterNoreturn),
244 PMW);
245 }
246}
247
248Error NVPTXCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
249 addFunctionPass(Pass: NVPTXLowerAggrCopiesPass(), PMW);
250 addFunctionPass(Pass: NVPTXAllocaHoistingPass(), PMW);
251 addMachineFunctionPass(Pass: NVPTXISelDAGToDAGPass(getTM(), getOptLevel()), PMW);
252 addMachineFunctionPass(Pass: NVPTXReplaceImageHandlesPass(), PMW);
253 return Error::success();
254}
255
256void NVPTXCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
257 addMachineFunctionPass(Pass: NVPTXForwardParamsPass(), PMW);
258 if (getOptLevel() != CodeGenOptLevel::None)
259 addMachineFunctionPass(Pass: NVPTXAddressFolderPass(), PMW);
260 // Remove Proxy Register pseudo instructions used to keep `callseq_end` alive.
261 addMachineFunctionPass(Pass: NVPTXProxyRegErasurePass(), PMW);
262}
263
264void NVPTXCodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) {
265 addMachineFunctionPass(Pass: NVPTXPrologEpilogPass(), PMW);
266 if (getOptLevel() != CodeGenOptLevel::None) {
267 // NVPTXPrologEpilogPass calculates frame object offset and replaces frame
268 // index with VRFrame register. NVPTXPeephole needs to be run after that
269 // and will replace VRFrame with VRFrameLocal when possible.
270 addMachineFunctionPass(Pass: NVPTXPeepholePass(), PMW);
271 }
272}
273
274Error NVPTXCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
275 addMachineFunctionPass(Pass: PHIEliminationPass(), PMW);
276 addMachineFunctionPass(Pass: TwoAddressInstructionPass(), PMW);
277 return Error::success();
278}
279
280Error NVPTXCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
281 addMachineFunctionPass(Pass: ProcessImplicitDefsPass(), PMW);
282 addMachineFunctionPass(
283 Pass: RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>(), PMW);
284 addMachineFunctionPass(Pass: PHIEliminationPass(), PMW);
285
286 addMachineFunctionPass(Pass: TwoAddressInstructionPass(), PMW);
287 addMachineFunctionPass(Pass: RegisterCoalescerPass(), PMW);
288
289 // PreRA instruction scheduling.
290 addMachineFunctionPass(Pass: MachineSchedulerPass(&TM), PMW);
291
292 addMachineFunctionPass(Pass: StackSlotColoringPass(), PMW);
293
294 // FIXME: Needs physical registers
295 // addMachineFunctionPass(MachineLICMPass(), PMW);
296
297 return Error::success();
298}
299
300void NVPTXCodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
301 addModulePass(Pass: NVPTXAsmPrinterBeginPass(), PMW, /*Force=*/true);
302}
303
304void NVPTXCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
305 addMachineFunctionPass(Pass: NVPTXAsmPrinterPass(), PMW);
306}
307
308void NVPTXCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
309 addModulePass(Pass: NVPTXAsmPrinterEndPass(), PMW);
310}
311
312} // namespace
313
314void NVPTXTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
315#define GET_PASS_REGISTRY "NVPTXPassRegistry.def"
316#include "llvm/Passes/TargetPassRegistry.inc"
317
318 PB.registerPipelineStartEPCallback(
319 C: [this](ModulePassManager &PM, OptimizationLevel Level) {
320 // We do not want to fold out calls to nvvm.reflect early if the user
321 // has not provided a target architecture just yet.
322 if (Subtarget.hasTargetName())
323 PM.addPass(Pass: NVVMReflectPass(Subtarget.getSmVersion()));
324
325 FunctionPassManager FPM;
326 // Note: NVVMIntrRangePass was causing numerical discrepancies at one
327 // point, if issues crop up, consider disabling.
328 FPM.addPass(Pass: NVVMIntrRangePass());
329 if (EarlyByValArgsCopy)
330 FPM.addPass(Pass: NVPTXCopyByValArgsPass());
331 PM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
332 });
333
334 if (!NoKernelInfoEndLTO) {
335 PB.registerFullLinkTimeOptimizationLastEPCallback(
336 C: [this](ModulePassManager &PM, OptimizationLevel Level) {
337 FunctionPassManager FPM;
338 FPM.addPass(Pass: KernelInfoPrinter(this));
339 PM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
340 });
341 }
342}
343
344Error NVPTXTargetMachine::buildCodeGenPipeline(
345 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
346 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
347 const CGPassBuilderOption &Opt, MCContext &Ctx,
348 PassInstrumentationCallbacks *PIC) {
349 auto CGPB = NVPTXCodeGenPassBuilder(*this, Opt, PIC);
350 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
351}
352