1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file contains the NVPTX CodeGen pipeline builder. It mirrors
10/// NVPTXPassConfig in NVPTXTargetMachine.cpp; the two must be kept in sync
11/// until the legacy pass manager path is removed.
12//===----------------------------------------------------------------------===//
13
14#include "NVPTX.h"
15#include "NVPTXAliasAnalysis.h"
16#include "NVPTXAsmPrinter.h"
17#include "NVPTXSubtarget.h"
18#include "NVPTXTargetMachine.h"
19#include "llvm/Analysis/KernelInfo.h"
20#include "llvm/CodeGen/AtomicExpand.h"
21#include "llvm/CodeGen/FuncletLayout.h"
22#include "llvm/CodeGen/LiveVariables.h"
23#include "llvm/CodeGen/MachineCopyPropagation.h"
24#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
25#include "llvm/CodeGen/MachineLoopInfo.h"
26#include "llvm/CodeGen/MachineScheduler.h"
27#include "llvm/CodeGen/PEI.h"
28#include "llvm/CodeGen/PHIElimination.h"
29#include "llvm/CodeGen/PatchableFunction.h"
30#include "llvm/CodeGen/PostRAMachineSink.h"
31#include "llvm/CodeGen/PostRASchedulerList.h"
32#include "llvm/CodeGen/ProcessImplicitDefs.h"
33#include "llvm/CodeGen/RegisterCoalescerPass.h"
34#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
35#include "llvm/CodeGen/ShrinkWrap.h"
36#include "llvm/CodeGen/StackSlotColoring.h"
37#include "llvm/CodeGen/TailDuplication.h"
38#include "llvm/CodeGen/TwoAddressInstructionPass.h"
39#include "llvm/CodeGen/UnreachableBlockElim.h"
40#include "llvm/IR/PassInstrumentation.h"
41#include "llvm/MC/MCStreamer.h"
42#include "llvm/Passes/CodeGenPassBuilder.h"
43#include "llvm/Passes/PassBuilder.h"
44#include "llvm/Target/CGPassBuilderOption.h"
45#include "llvm/Transforms/IPO/ExpandVariadics.h"
46#include "llvm/Transforms/Scalar/EarlyCSE.h"
47#include "llvm/Transforms/Scalar/GVN.h"
48#include "llvm/Transforms/Scalar/InferAddressSpaces.h"
49#include "llvm/Transforms/Scalar/NaryReassociate.h"
50#include "llvm/Transforms/Scalar/SROA.h"
51#include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"
52#include "llvm/Transforms/Scalar/SpeculativeExecution.h"
53#include "llvm/Transforms/Scalar/StraightLineStrengthReduce.h"
54#include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
55
56using namespace llvm;
57
58extern cl::opt<bool> DisableLoadStoreVectorizer;
59extern cl::opt<bool> DisableNVPTXIRPeephole;
60
61// byval arguments in NVPTX are special. We're only allowed to read from them
62// using a special instruction, and if we ever need to write to them or take an
63// address, we must make a local copy and use it, instead.
64//
65// The problem is that local copies are very expensive, and we create them very
66// late in the compilation pipeline, so LLVM does not have much of a chance to
67// eliminate them, if they turn out to be unnecessary.
68//
69// One way around that is to create such copies early on, and let them percolate
70// through the optimizations. The copying itself will never trigger creation of
71// another copy later on, as the reads are allowed. If LLVM can eliminate it,
72// it's a win. It the full optimization pipeline can't remove the copy, that's
73// as good as it gets in terms of the effort we could've done, and it's
74// certainly a much better effort than what we do now.
75//
76// This early injection of the copies has potential to create undesireable
77// side-effects, so it's disabled by default, for now, until it sees more
78// testing.
79static cl::opt<bool> EarlyByValArgsCopy(
80 "nvptx-early-byval-copy",
81 cl::desc("Create a copy of byval function arguments early."),
82 cl::init(Val: false), cl::Hidden);
83
84namespace {
85
86class NVPTXCodeGenPassBuilder : public CodeGenPassBuilder {
87 using Base = CodeGenPassBuilder;
88
89 NVPTXTargetMachine &getTM() const {
90 return static_cast<NVPTXTargetMachine &>(TM);
91 }
92
93public:
94 explicit NVPTXCodeGenPassBuilder(NVPTXTargetMachine &TM,
95 const CGPassBuilderOption &Opts,
96 PassInstrumentationCallbacks *PIC)
97 : CodeGenPassBuilder(TM, Opts, PIC) {
98 // The following passes are known to not play well with virtual regs
99 // hanging around after register allocation (which in our case, is *all*
100 // registers). We explicitly disable them here. We do, however, need some
101 // functionality of the PrologEpilogCodeInserter pass, so we emulate that
102 // behavior in the NVPTXPrologEpilog pass (see NVPTXPrologEpilogPass.cpp).
103 disablePass<PrologEpilogInserterPass, MachineLateInstrsCleanupPass,
104 MachineCopyPropagationPass, TailDuplicatePass,
105 StackMapLivenessPass, PostRAMachineSinkingPass,
106 PostRASchedulerPass, FuncletLayoutPass, PatchableFunctionPass,
107 ShrinkWrapPass, RemoveLoadsIntoFakeUsesPass>();
108 }
109
110 void addIRPasses(PassManagerWrapper &PMW) override;
111 Error addInstSelector(PassManagerWrapper &PMW) override;
112 void addPreRegAlloc(PassManagerWrapper &PMW) override;
113 void addPostRegAlloc(PassManagerWrapper &PMW) override;
114
115 // NVPTX has no register allocation; virtual registers are emitted directly.
116 void addTargetRegisterAllocator(PassManagerWrapper &PMW, bool) override {}
117 Error addFastRegAlloc(PassManagerWrapper &PMW) override;
118 Error addOptimizedRegAlloc(PassManagerWrapper &PMW) override;
119
120 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
121 void addAsmPrinter(PassManagerWrapper &PMW) override;
122 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
123
124private:
125 // If the opt level is aggressive, add GVN; otherwise, add EarlyCSE.
126 void addEarlyCSEOrGVNPass(PassManagerWrapper &PMW);
127
128 // Add passes that propagate special memory spaces.
129 void addAddressSpaceInferencePasses(PassManagerWrapper &PMW);
130
131 // Add passes that perform straight-line scalar optimizations.
132 void addStraightLineScalarOptimizationPasses(PassManagerWrapper &PMW);
133};
134
135void NVPTXCodeGenPassBuilder::addEarlyCSEOrGVNPass(PassManagerWrapper &PMW) {
136 if (getOptLevel() == CodeGenOptLevel::Aggressive)
137 // Disable scalar PRE due to Register Pressure increase
138 addFunctionPass(Pass: GVNPass(GVNOptions().setScalarPRE(false)), PMW);
139 else
140 addFunctionPass(Pass: EarlyCSEPass(), PMW);
141}
142
143void NVPTXCodeGenPassBuilder::addAddressSpaceInferencePasses(
144 PassManagerWrapper &PMW) {
145 // NVPTXLowerArgs emits alloca for byval parameters which can often
146 // be eliminated by SROA.
147 addFunctionPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
148 /*AggregateToVector=*/true)),
149 PMW);
150 addFunctionPass(Pass: NVPTXLowerAllocaPass(), PMW);
151 // TODO: Consider running InferAddressSpaces during opt, earlier in the
152 // compilation flow.
153 addFunctionPass(Pass: InferAddressSpacesPass(), PMW);
154 addFunctionPass(Pass: NVPTXAtomicLowerPass(), PMW);
155}
156
157void NVPTXCodeGenPassBuilder::addStraightLineScalarOptimizationPasses(
158 PassManagerWrapper &PMW) {
159 addFunctionPass(Pass: SeparateConstOffsetFromGEPPass(), PMW);
160 addFunctionPass(Pass: SpeculativeExecutionPass(), PMW);
161 // ReassociateGEPs exposes more opportunites for SLSR. See
162 // the example in reassociate-geps-and-slsr.ll.
163 addFunctionPass(Pass: StraightLineStrengthReducePass(), PMW);
164 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN
165 // or EarlyCSE can reuse. GVN generates significantly better code than
166 // EarlyCSE for some of our benchmarks.
167 addEarlyCSEOrGVNPass(PMW);
168 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
169 addFunctionPass(Pass: NaryReassociatePass(), PMW);
170 // NaryReassociate on GEPs creates redundant common expressions, so run
171 // EarlyCSE after it.
172 addFunctionPass(Pass: EarlyCSEPass(), PMW);
173}
174
175void NVPTXCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
176 const NVPTXSubtarget &ST = *getTM().getSubtargetImpl();
177
178 // NVVMReflectPass is added in the pipeline-start extension point, so
179 // hopefully running it here does nothing. But since we need it for
180 // correctness when lowering to NVPTX, run it here too, in case whoever built
181 // our pass pipeline didn't add it.
182 flushFPMsToMPM(PMW);
183 addModulePass(Pass: NVVMReflectPass(ST.getSmVersion()), PMW);
184
185 if (getOptLevel() != CodeGenOptLevel::None)
186 addFunctionPass(Pass: NVPTXImageOptimizerPass(), PMW);
187 flushFPMsToMPM(PMW);
188 addModulePass(Pass: NVPTXAssignValidGlobalNamesPass(), PMW);
189 addModulePass(Pass: GenericToNVVMPass(), PMW);
190
191 // Lower variadic calls before address space inference.
192 addModulePass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Lowering), PMW);
193
194 // NVPTXLowerArgs is required for correctness and should be run right
195 // before the address space inference passes.
196 if (getTM().getDrvInterface() == NVPTX::CUDA) {
197 addFunctionPass(Pass: NVPTXMarkKernelPtrsGlobalPass(), PMW);
198 flushFPMsToMPM(PMW);
199 }
200 addModulePass(Pass: NVPTXPromoteParamAlignPass(), PMW);
201 addModulePass(Pass: NVPTXLowerArgsPass(TM), PMW);
202 if (getOptLevel() != CodeGenOptLevel::None) {
203 addAddressSpaceInferencePasses(PMW);
204 addStraightLineScalarOptimizationPasses(PMW);
205 } else {
206 // Required for correct stack lowering
207 addFunctionPass(Pass: NVPTXLowerAllocaPass(), PMW);
208 }
209
210 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
211 flushFPMsToMPM(PMW);
212 addModulePass(Pass: NVPTXCtorDtorLoweringPass(), PMW);
213
214 // === LSR and other generic IR passes ===
215 Base::addIRPasses(PMW);
216 // EarlyCSE is not always strong enough to clean up what LSR produces. For
217 // example, GVN can combine
218 //
219 // %0 = add %a, %b
220 // %1 = add %b, %a
221 //
222 // and
223 //
224 // %0 = shl nsw %a, 2
225 // %1 = shl %a, 2
226 //
227 // but EarlyCSE can do neither of them.
228 if (getOptLevel() != CodeGenOptLevel::None) {
229 addEarlyCSEOrGVNPass(PMW);
230 if (!DisableLoadStoreVectorizer)
231 addFunctionPass(Pass: LoadStoreVectorizerPass(), PMW);
232 addFunctionPass(Pass: SROAPass(SROAOptions(SROAOptions::PreserveCFG,
233 /*AggregateToVector=*/true)),
234 PMW);
235 addFunctionPass(Pass: NVPTXTagInvariantLoadsPass(), PMW);
236 if (!DisableNVPTXIRPeephole)
237 addFunctionPass(Pass: NVPTXIRPeepholePass(), PMW);
238 }
239
240 if (ST.hasPTXASUnreachableBug()) {
241 // Run LowerUnreachable to WAR a ptxas bug. See the commit description of
242 // 1ee4d880e8760256c606fe55b7af85a4f70d006d for more details.
243 addFunctionPass(Pass: NVPTXLowerUnreachablePass(TM.Options.TrapUnreachable,
244 TM.Options.NoTrapAfterNoreturn),
245 PMW);
246 }
247}
248
249Error NVPTXCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
250 addFunctionPass(Pass: NVPTXLowerAggrCopiesPass(), PMW);
251 addFunctionPass(Pass: NVPTXAllocaHoistingPass(), PMW);
252 addMachineFunctionPass(Pass: NVPTXISelDAGToDAGPass(getTM(), getOptLevel()), PMW);
253 addMachineFunctionPass(Pass: NVPTXReplaceImageHandlesPass(), PMW);
254 return Error::success();
255}
256
257void NVPTXCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
258 addMachineFunctionPass(Pass: NVPTXForwardParamsPass(), PMW);
259 if (getOptLevel() != CodeGenOptLevel::None)
260 addMachineFunctionPass(Pass: NVPTXAddressFolderPass(), PMW);
261 // Remove Proxy Register pseudo instructions used to keep `callseq_end` alive.
262 addMachineFunctionPass(Pass: NVPTXProxyRegErasurePass(), PMW);
263}
264
265void NVPTXCodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) {
266 addMachineFunctionPass(Pass: NVPTXPrologEpilogPass(), PMW);
267 if (getOptLevel() != CodeGenOptLevel::None) {
268 // NVPTXPrologEpilogPass calculates frame object offset and replaces frame
269 // index with VRFrame register. NVPTXPeephole needs to be run after that
270 // and will replace VRFrame with VRFrameLocal when possible.
271 addMachineFunctionPass(Pass: NVPTXPeepholePass(), PMW);
272 }
273}
274
275Error NVPTXCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
276 addMachineFunctionPass(Pass: PHIEliminationPass(), PMW);
277 addMachineFunctionPass(Pass: TwoAddressInstructionPass(), PMW);
278 return Error::success();
279}
280
281Error NVPTXCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
282 addMachineFunctionPass(Pass: ProcessImplicitDefsPass(), PMW);
283 // LiveVariables requires pure SSA form and no unreachable blocks; the legacy
284 // pass manager pulls UnreachableMachineBlockElim in as an implicit
285 // dependency, so add it explicitly here.
286 addMachineFunctionPass(Pass: UnreachableMachineBlockElimPass(), PMW);
287 addMachineFunctionPass(
288 Pass: RequireAnalysisPass<LiveVariablesAnalysis, MachineFunction>(), PMW);
289 addMachineFunctionPass(
290 Pass: RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>(), PMW);
291 addMachineFunctionPass(Pass: PHIEliminationPass(), PMW);
292
293 addMachineFunctionPass(Pass: TwoAddressInstructionPass(), PMW);
294 addMachineFunctionPass(Pass: RegisterCoalescerPass(), PMW);
295
296 // PreRA instruction scheduling.
297 addMachineFunctionPass(Pass: MachineSchedulerPass(&TM), PMW);
298
299 addMachineFunctionPass(Pass: StackSlotColoringPass(), PMW);
300
301 // FIXME: Needs physical registers
302 // addMachineFunctionPass(MachineLICMPass(), PMW);
303
304 return Error::success();
305}
306
307void NVPTXCodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
308 addModulePass(Pass: NVPTXAsmPrinterBeginPass(), PMW, /*Force=*/true);
309}
310
311void NVPTXCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
312 addMachineFunctionPass(Pass: NVPTXAsmPrinterPass(), PMW);
313}
314
315void NVPTXCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
316 addModulePass(Pass: NVPTXAsmPrinterEndPass(), PMW);
317}
318
319} // namespace
320
321void NVPTXTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
322#define GET_PASS_REGISTRY "NVPTXPassRegistry.def"
323#include "llvm/Passes/TargetPassRegistry.inc"
324
325 PB.registerPipelineStartEPCallback(
326 C: [this](ModulePassManager &PM, OptimizationLevel Level) {
327 // We do not want to fold out calls to nvvm.reflect early if the user
328 // has not provided a target architecture just yet.
329 if (Subtarget.hasTargetName())
330 PM.addPass(Pass: NVVMReflectPass(Subtarget.getSmVersion()));
331
332 FunctionPassManager FPM;
333 // Note: NVVMIntrRangePass was causing numerical discrepancies at one
334 // point, if issues crop up, consider disabling.
335 FPM.addPass(Pass: NVVMIntrRangePass());
336 if (EarlyByValArgsCopy)
337 FPM.addPass(Pass: NVPTXCopyByValArgsPass());
338 PM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
339 });
340
341 if (!NoKernelInfoEndLTO) {
342 PB.registerFullLinkTimeOptimizationLastEPCallback(
343 C: [this](ModulePassManager &PM, OptimizationLevel Level) {
344 FunctionPassManager FPM;
345 FPM.addPass(Pass: KernelInfoPrinter(this));
346 PM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
347 });
348 }
349}
350
351Error NVPTXTargetMachine::buildCodeGenPipeline(
352 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
353 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
354 const CGPassBuilderOption &Opt, MCContext &Ctx,
355 PassInstrumentationCallbacks *PIC) {
356 auto CGPB = NVPTXCodeGenPassBuilder(*this, Opt, PIC);
357 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
358}
359