1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file contains the NVPTX CodeGen pipeline builder. It mirrors
10/// NVPTXPassConfig in NVPTXTargetMachine.cpp; the two must be kept in sync
11/// until the legacy pass manager path is removed.
12//===----------------------------------------------------------------------===//
13
14#include "NVPTX.h"
15#include "NVPTXAliasAnalysis.h"
16#include "NVPTXAsmPrinter.h"
17#include "NVPTXSubtarget.h"
18#include "NVPTXTargetMachine.h"
19#include "llvm/Analysis/KernelInfo.h"
20#include "llvm/CodeGen/AtomicExpand.h"
21#include "llvm/CodeGen/DeadMachineInstructionElim.h"
22#include "llvm/CodeGen/FuncletLayout.h"
23#include "llvm/CodeGen/LiveVariables.h"
24#include "llvm/CodeGen/MachineCopyPropagation.h"
25#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
26#include "llvm/CodeGen/MachineLoopInfo.h"
27#include "llvm/CodeGen/MachineScheduler.h"
28#include "llvm/CodeGen/PEI.h"
29#include "llvm/CodeGen/PHIElimination.h"
30#include "llvm/CodeGen/PatchableFunction.h"
31#include "llvm/CodeGen/PostRAMachineSink.h"
32#include "llvm/CodeGen/PostRASchedulerList.h"
33#include "llvm/CodeGen/ProcessImplicitDefs.h"
34#include "llvm/CodeGen/RegisterCoalescerPass.h"
35#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
36#include "llvm/CodeGen/ShrinkWrap.h"
37#include "llvm/CodeGen/StackColoring.h"
38#include "llvm/CodeGen/StackSlotColoring.h"
39#include "llvm/CodeGen/TailDuplication.h"
40#include "llvm/CodeGen/TwoAddressInstructionPass.h"
41#include "llvm/CodeGen/UnreachableBlockElim.h"
42#include "llvm/IR/PassInstrumentation.h"
43#include "llvm/MC/MCStreamer.h"
44#include "llvm/Passes/CodeGenPassBuilder.h"
45#include "llvm/Passes/PassBuilder.h"
46#include "llvm/Target/CGPassBuilderOption.h"
47#include "llvm/Transforms/IPO/ExpandVariadics.h"
48#include "llvm/Transforms/Scalar/EarlyCSE.h"
49#include "llvm/Transforms/Scalar/GVN.h"
50#include "llvm/Transforms/Scalar/InferAddressSpaces.h"
51#include "llvm/Transforms/Scalar/NaryReassociate.h"
52#include "llvm/Transforms/Scalar/SROA.h"
53#include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"
54#include "llvm/Transforms/Scalar/SpeculativeExecution.h"
55#include "llvm/Transforms/Scalar/StraightLineStrengthReduce.h"
56#include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
57
58using namespace llvm;
59
60extern cl::opt<bool> DisableLoadStoreVectorizer;
61extern cl::opt<bool> DisableNVPTXIRPeephole;
62
63// byval arguments in NVPTX are special. We're only allowed to read from them
64// using a special instruction, and if we ever need to write to them or take an
65// address, we must make a local copy and use it, instead.
66//
67// The problem is that local copies are very expensive, and we create them very
68// late in the compilation pipeline, so LLVM does not have much of a chance to
69// eliminate them, if they turn out to be unnecessary.
70//
71// One way around that is to create such copies early on, and let them percolate
72// through the optimizations. The copying itself will never trigger creation of
73// another copy later on, as the reads are allowed. If LLVM can eliminate it,
74// it's a win. It the full optimization pipeline can't remove the copy, that's
75// as good as it gets in terms of the effort we could've done, and it's
76// certainly a much better effort than what we do now.
77//
78// This early injection of the copies has potential to create undesireable
79// side-effects, so it's disabled by default, for now, until it sees more
80// testing.
81static cl::opt<bool> EarlyByValArgsCopy(
82 "nvptx-early-byval-copy",
83 cl::desc("Create a copy of byval function arguments early."),
84 cl::init(Val: false), cl::Hidden);
85
86namespace {
87
88class NVPTXCodeGenPassBuilder : public CodeGenPassBuilder {
89 using Base = CodeGenPassBuilder;
90
91 NVPTXTargetMachine &getTM() const {
92 return static_cast<NVPTXTargetMachine &>(TM);
93 }
94
95public:
96 explicit NVPTXCodeGenPassBuilder(NVPTXTargetMachine &TM,
97 const CGPassBuilderOption &Opts,
98 PassInstrumentationCallbacks *PIC)
99 : CodeGenPassBuilder(TM, Opts, PIC) {
100 // The following passes are known to not play well with virtual regs
101 // hanging around after register allocation (which in our case, is *all*
102 // registers). We explicitly disable them here. We do, however, need some
103 // functionality of the PrologEpilogCodeInserter pass, so we emulate that
104 // behavior in the NVPTXPrologEpilog pass (see NVPTXPrologEpilogPass.cpp).
105 disablePass<PrologEpilogInserterPass, MachineLateInstrsCleanupPass,
106 MachineCopyPropagationPass, TailDuplicatePass,
107 StackMapLivenessPass, PostRAMachineSinkingPass,
108 PostRASchedulerPass, FuncletLayoutPass, PatchableFunctionPass,
109 ShrinkWrapPass, RemoveLoadsIntoFakeUsesPass>();
110 }
111
112 void addIRPasses(PassManagerWrapper &PMW) override;
113 Error addInstSelector(PassManagerWrapper &PMW) override;
114 void addPreRegAlloc(PassManagerWrapper &PMW) override;
115 void addPostRegAlloc(PassManagerWrapper &PMW) override;
116
117 // NVPTX has no register allocation; virtual registers are emitted directly.
118 void addTargetRegisterAllocator(PassManagerWrapper &PMW, bool) override {}
119 Error addFastRegAlloc(PassManagerWrapper &PMW) override;
120 Error addOptimizedRegAlloc(PassManagerWrapper &PMW) override;
121
122 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
123 void addAsmPrinter(PassManagerWrapper &PMW) override;
124 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
125
126private:
127 // If the opt level is aggressive, add GVN; otherwise, add EarlyCSE.
128 void addEarlyCSEOrGVNPass(PassManagerWrapper &PMW);
129
130 // Add passes that propagate special memory spaces.
131 void addAddressSpaceInferencePasses(PassManagerWrapper &PMW);
132
133 // Add passes that perform straight-line scalar optimizations.
134 void addStraightLineScalarOptimizationPasses(PassManagerWrapper &PMW);
135};
136
137void NVPTXCodeGenPassBuilder::addEarlyCSEOrGVNPass(PassManagerWrapper &PMW) {
138 if (getOptLevel() == CodeGenOptLevel::Aggressive)
139 // Disable scalar PRE due to Register Pressure increase
140 addFunctionPass(Pass: GVNPass(GVNOptions().setScalarPRE(false)), PMW);
141 else
142 addFunctionPass(Pass: EarlyCSEPass(), PMW);
143}
144
145void NVPTXCodeGenPassBuilder::addAddressSpaceInferencePasses(
146 PassManagerWrapper &PMW) {
147 // NVPTXLowerArgs emits alloca for byval parameters which can often
148 // be eliminated by SROA.
149 addFunctionPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
150 /*AggregateToVector=*/true)),
151 PMW);
152 addFunctionPass(Pass: NVPTXLowerAllocaPass(), PMW);
153 // TODO: Consider running InferAddressSpaces during opt, earlier in the
154 // compilation flow.
155 addFunctionPass(Pass: InferAddressSpacesPass(), PMW);
156 addFunctionPass(Pass: NVPTXAtomicLowerPass(), PMW);
157}
158
159void NVPTXCodeGenPassBuilder::addStraightLineScalarOptimizationPasses(
160 PassManagerWrapper &PMW) {
161 addFunctionPass(Pass: SeparateConstOffsetFromGEPPass(), PMW);
162 addFunctionPass(Pass: SpeculativeExecutionPass(), PMW);
163 // ReassociateGEPs exposes more opportunites for SLSR. See
164 // the example in reassociate-geps-and-slsr.ll.
165 addFunctionPass(Pass: StraightLineStrengthReducePass(), PMW);
166 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN
167 // or EarlyCSE can reuse. GVN generates significantly better code than
168 // EarlyCSE for some of our benchmarks.
169 addEarlyCSEOrGVNPass(PMW);
170 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
171 addFunctionPass(Pass: NaryReassociatePass(), PMW);
172 // NaryReassociate on GEPs creates redundant common expressions, so run
173 // EarlyCSE after it.
174 addFunctionPass(Pass: EarlyCSEPass(), PMW);
175}
176
177void NVPTXCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
178 const NVPTXSubtarget &ST = *getTM().getSubtargetImpl();
179
180 // NVVMReflectPass is added in the pipeline-start extension point, so
181 // hopefully running it here does nothing. But since we need it for
182 // correctness when lowering to NVPTX, run it here too, in case whoever built
183 // our pass pipeline didn't add it.
184 flushFPMsToMPM(PMW);
185 addModulePass(Pass: NVVMReflectPass(ST.getSmVersion()), PMW);
186
187 if (getOptLevel() != CodeGenOptLevel::None)
188 addFunctionPass(Pass: NVPTXImageOptimizerPass(), PMW);
189 flushFPMsToMPM(PMW);
190 addModulePass(Pass: NVPTXAssignValidGlobalNamesPass(), PMW);
191 addModulePass(Pass: GenericToNVVMPass(), PMW);
192
193 // Lower variadic calls before address space inference.
194 addModulePass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Lowering), PMW);
195
196 // NVPTXLowerArgs is required for correctness and should be run right
197 // before the address space inference passes.
198 if (getTM().getDrvInterface() == NVPTX::CUDA) {
199 addFunctionPass(Pass: NVPTXMarkKernelPtrsGlobalPass(), PMW);
200 flushFPMsToMPM(PMW);
201 }
202 addModulePass(Pass: NVPTXPromoteParamAlignPass(), PMW);
203 addModulePass(Pass: NVPTXLowerArgsPass(TM), PMW);
204 if (getOptLevel() != CodeGenOptLevel::None) {
205 addAddressSpaceInferencePasses(PMW);
206 addStraightLineScalarOptimizationPasses(PMW);
207 } else {
208 // Required for correct stack lowering
209 addFunctionPass(Pass: NVPTXLowerAllocaPass(), PMW);
210 }
211
212 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
213 flushFPMsToMPM(PMW);
214 addModulePass(Pass: NVPTXCtorDtorLoweringPass(), PMW);
215
216 // === LSR and other generic IR passes ===
217 Base::addIRPasses(PMW);
218 // EarlyCSE is not always strong enough to clean up what LSR produces. For
219 // example, GVN can combine
220 //
221 // %0 = add %a, %b
222 // %1 = add %b, %a
223 //
224 // and
225 //
226 // %0 = shl nsw %a, 2
227 // %1 = shl %a, 2
228 //
229 // but EarlyCSE can do neither of them.
230 if (getOptLevel() != CodeGenOptLevel::None) {
231 addEarlyCSEOrGVNPass(PMW);
232 if (!DisableLoadStoreVectorizer)
233 addFunctionPass(Pass: LoadStoreVectorizerPass(), PMW);
234 addFunctionPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
235 /*AggregateToVector=*/true)),
236 PMW);
237 addFunctionPass(Pass: NVPTXTagInvariantLoadsPass(), PMW);
238 if (!DisableNVPTXIRPeephole)
239 addFunctionPass(Pass: NVPTXIRPeepholePass(), PMW);
240 }
241
242 if (ST.hasPTXASUnreachableBug()) {
243 // Run LowerUnreachable to WAR a ptxas bug. See the commit description of
244 // 1ee4d880e8760256c606fe55b7af85a4f70d006d for more details.
245 addFunctionPass(Pass: NVPTXLowerUnreachablePass(TM.Options.TrapUnreachable,
246 TM.Options.NoTrapAfterNoreturn),
247 PMW);
248 }
249}
250
251Error NVPTXCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
252 addFunctionPass(Pass: NVPTXLowerAggrCopiesPass(), PMW);
253 addFunctionPass(Pass: NVPTXAllocaHoistingPass(), PMW);
254 addMachineFunctionPass(Pass: NVPTXISelDAGToDAGPass(getTM(), getOptLevel()), PMW);
255 addMachineFunctionPass(Pass: NVPTXReplaceImageHandlesPass(), PMW);
256 return Error::success();
257}
258
259void NVPTXCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
260 addMachineFunctionPass(Pass: NVPTXForwardParamsPass(), PMW);
261 if (getOptLevel() != CodeGenOptLevel::None)
262 addMachineFunctionPass(Pass: NVPTXAddressFolderPass(), PMW);
263 // Remove Proxy Register pseudo instructions used to keep `callseq_end` alive.
264 addMachineFunctionPass(Pass: NVPTXProxyRegErasurePass(), PMW);
265}
266
267void NVPTXCodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) {
268 addMachineFunctionPass(Pass: NVPTXPrologEpilogPass(), PMW);
269 if (getOptLevel() != CodeGenOptLevel::None) {
270 // NVPTXPrologEpilogPass calculates frame object offset and replaces frame
271 // index with VRFrame register. NVPTXPeephole needs to be run after that
272 // and will replace VRFrame with VRFrameLocal when possible.
273 addMachineFunctionPass(Pass: NVPTXPeepholePass(), PMW);
274 }
275}
276
277Error NVPTXCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
278 addMachineFunctionPass(Pass: PHIEliminationPass(), PMW);
279 addMachineFunctionPass(Pass: TwoAddressInstructionPass(), PMW);
280 return Error::success();
281}
282
283Error NVPTXCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
284 addMachineFunctionPass(Pass: ProcessImplicitDefsPass(), PMW);
285 // LiveVariables requires pure SSA form and no unreachable blocks; the legacy
286 // pass manager pulls UnreachableMachineBlockElim in as an implicit
287 // dependency, so add it explicitly here.
288 addMachineFunctionPass(Pass: UnreachableMachineBlockElimPass(), PMW);
289 addMachineFunctionPass(
290 Pass: RequireAnalysisPass<LiveVariablesAnalysis, MachineFunction>(), PMW);
291 addMachineFunctionPass(
292 Pass: RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>(), PMW);
293 addMachineFunctionPass(Pass: PHIEliminationPass(), PMW);
294
295 addMachineFunctionPass(Pass: TwoAddressInstructionPass(), PMW);
296 addMachineFunctionPass(Pass: RegisterCoalescerPass(), PMW);
297
298 // PreRA instruction scheduling.
299 addMachineFunctionPass(Pass: MachineSchedulerPass(&TM), PMW);
300
301 addMachineFunctionPass(Pass: StackSlotColoringPass(), PMW);
302
303 // FIXME: Needs physical registers
304 // addMachineFunctionPass(MachineLICMPass(), PMW);
305
306 return Error::success();
307}
308
309void NVPTXCodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
310 addModulePass(Pass: NVPTXAsmPrinterBeginPass(), PMW, /*Force=*/true);
311}
312
313void NVPTXCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
314 addMachineFunctionPass(Pass: NVPTXAsmPrinterPass(), PMW);
315}
316
317void NVPTXCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
318 addModulePass(Pass: NVPTXAsmPrinterEndPass(), PMW);
319}
320
321} // namespace
322
323void NVPTXTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
324#define GET_PASS_REGISTRY "NVPTXPassRegistry.def"
325#include "llvm/Passes/TargetPassRegistry.inc"
326
327 PB.registerPipelineStartEPCallback(
328 C: [this](ModulePassManager &PM, OptimizationLevel Level) {
329 // We do not want to fold out calls to nvvm.reflect early if the user
330 // has not provided a target architecture just yet.
331 if (Subtarget.hasTargetName())
332 PM.addPass(Pass: NVVMReflectPass(Subtarget.getSmVersion()));
333
334 FunctionPassManager FPM;
335 // Note: NVVMIntrRangePass was causing numerical discrepancies at one
336 // point, if issues crop up, consider disabling.
337 FPM.addPass(Pass: NVVMIntrRangePass());
338 if (EarlyByValArgsCopy)
339 FPM.addPass(Pass: NVPTXCopyByValArgsPass());
340 PM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
341 });
342
343 if (!NoKernelInfoEndLTO) {
344 PB.registerFullLinkTimeOptimizationLastEPCallback(
345 C: [this](ModulePassManager &PM, OptimizationLevel Level) {
346 FunctionPassManager FPM;
347 FPM.addPass(Pass: KernelInfoPrinter(this));
348 PM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
349 });
350 }
351}
352
353Error NVPTXTargetMachine::buildCodeGenPipeline(
354 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
355 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
356 const CGPassBuilderOption &Opt, MCContext &Ctx,
357 PassInstrumentationCallbacks *PIC) {
358 auto CGPB = NVPTXCodeGenPassBuilder(*this, Opt, PIC);
359 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
360}
361