1//===-- AMDGPUTargetMachine.cpp - TargetMachine for hw codegen targets-----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file contains both AMDGPU target machine and the CodeGen pass builder.
11/// The AMDGPU target machine contains all of the hardware specific information
12/// needed to emit code for SI+ GPUs in the legacy pass manager pipeline. The
13/// CodeGen pass builder handles the pass pipeline for new pass manager.
14//
15//===----------------------------------------------------------------------===//
16
17#include "AMDGPUTargetMachine.h"
18#include "AMDGPU.h"
19#include "AMDGPUAliasAnalysis.h"
20#include "AMDGPUAsmPrinter.h"
21#include "AMDGPUBarrierLatency.h"
22#include "AMDGPUCoExecSchedStrategy.h"
23#include "AMDGPUCtorDtorLowering.h"
24#include "AMDGPUExportClustering.h"
25#include "AMDGPUExportKernelRuntimeHandles.h"
26#include "AMDGPUHazardLatency.h"
27#include "AMDGPUIGroupLP.h"
28#include "AMDGPUISelDAGToDAG.h"
29#include "AMDGPULowerVGPREncoding.h"
30#include "AMDGPUMacroFusion.h"
31#include "AMDGPUNextUseAnalysis.h"
32#include "AMDGPUPerfHintAnalysis.h"
33#include "AMDGPUPreloadKernArgProlog.h"
34#include "AMDGPUPrepareAGPRAlloc.h"
35#include "AMDGPURemoveIncompatibleFunctions.h"
36#include "AMDGPUReserveWWMRegs.h"
37#include "AMDGPUResourceUsageAnalysis.h"
38#include "AMDGPUSplitModule.h"
39#include "AMDGPUTargetObjectFile.h"
40#include "AMDGPUTargetTransformInfo.h"
41#include "AMDGPUUnifyDivergentExitNodes.h"
42#include "AMDGPUWaitSGPRHazards.h"
43#include "GCNDPPCombine.h"
44#include "GCNIterativeScheduler.h"
45#include "GCNNSAReassign.h"
46#include "GCNPreRALongBranchReg.h"
47#include "GCNPreRAOptimizations.h"
48#include "GCNRewritePartialRegUses.h"
49#include "GCNSchedStrategy.h"
50#include "GCNVOPDUtils.h"
51#include "R600.h"
52#include "R600TargetMachine.h"
53#include "SIFixSGPRCopies.h"
54#include "SIFixVGPRCopies.h"
55#include "SIFoldOperands.h"
56#include "SIFormMemoryClauses.h"
57#include "SILoadStoreOptimizer.h"
58#include "SILowerControlFlow.h"
59#include "SILowerSGPRSpills.h"
60#include "SILowerWWMCopies.h"
61#include "SIMachineFunctionInfo.h"
62#include "SIMachineScheduler.h"
63#include "SIOptimizeExecMasking.h"
64#include "SIOptimizeExecMaskingPreRA.h"
65#include "SIOptimizeVGPRLiveRange.h"
66#include "SIPeepholeSDWA.h"
67#include "SIPostRABundler.h"
68#include "SIPreAllocateWWMRegs.h"
69#include "SIShrinkInstructions.h"
70#include "SIWholeQuadMode.h"
71#include "TargetInfo/AMDGPUTargetInfo.h"
72#include "Utils/AMDGPUBaseInfo.h"
73#include "llvm/Analysis/CGSCCPassManager.h"
74#include "llvm/Analysis/CallGraphSCCPass.h"
75#include "llvm/Analysis/KernelInfo.h"
76#include "llvm/Analysis/UniformityAnalysis.h"
77#include "llvm/CodeGen/AtomicExpand.h"
78#include "llvm/CodeGen/BranchRelaxation.h"
79#include "llvm/CodeGen/DeadMachineInstructionElim.h"
80#include "llvm/CodeGen/EarlyIfConversion.h"
81#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
82#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
83#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
84#include "llvm/CodeGen/GlobalISel/Legalizer.h"
85#include "llvm/CodeGen/GlobalISel/Localizer.h"
86#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
87#include "llvm/CodeGen/MIRParser/MIParser.h"
88#include "llvm/CodeGen/MachineCSE.h"
89#include "llvm/CodeGen/MachineLICM.h"
90#include "llvm/CodeGen/MachineScheduler.h"
91#include "llvm/CodeGen/Passes.h"
92#include "llvm/CodeGen/PostRAHazardRecognizer.h"
93#include "llvm/CodeGen/RegAllocRegistry.h"
94#include "llvm/CodeGen/TargetPassConfig.h"
95#include "llvm/IR/DiagnosticInfo.h"
96#include "llvm/IR/IntrinsicsAMDGPU.h"
97#include "llvm/IR/Module.h"
98#include "llvm/IR/PassManager.h"
99#include "llvm/IR/PatternMatch.h"
100#include "llvm/InitializePasses.h"
101#include "llvm/MC/TargetRegistry.h"
102#include "llvm/Passes/CodeGenPassBuilder.h"
103#include "llvm/Passes/PassBuilder.h"
104#include "llvm/Support/Compiler.h"
105#include "llvm/Support/FormatVariadic.h"
106#include "llvm/TargetParser/AMDGPUTargetParser.h"
107#include "llvm/Transforms/HipStdPar/HipStdPar.h"
108#include "llvm/Transforms/IPO.h"
109#include "llvm/Transforms/IPO/AlwaysInliner.h"
110#include "llvm/Transforms/IPO/ExpandVariadics.h"
111#include "llvm/Transforms/IPO/GlobalDCE.h"
112#include "llvm/Transforms/IPO/Internalize.h"
113#include "llvm/Transforms/Scalar.h"
114#include "llvm/Transforms/Scalar/EarlyCSE.h"
115#include "llvm/Transforms/Scalar/FlattenCFG.h"
116#include "llvm/Transforms/Scalar/GVN.h"
117#include "llvm/Transforms/Scalar/InferAddressSpaces.h"
118#include "llvm/Transforms/Scalar/LICM.h"
119#include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
120#include "llvm/Transforms/Scalar/LoopPassManager.h"
121#include "llvm/Transforms/Scalar/NaryReassociate.h"
122#include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"
123#include "llvm/Transforms/Scalar/Sink.h"
124#include "llvm/Transforms/Scalar/StraightLineStrengthReduce.h"
125#include "llvm/Transforms/Scalar/StructurizeCFG.h"
126#include "llvm/Transforms/Utils.h"
127#include "llvm/Transforms/Utils/FixIrreducible.h"
128#include "llvm/Transforms/Utils/LCSSA.h"
129#include "llvm/Transforms/Utils/LowerSwitch.h"
130#include "llvm/Transforms/Utils/SimplifyLibCalls.h"
131#include "llvm/Transforms/Utils/UnifyLoopExits.h"
132#include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
133#include <optional>
134
135using namespace llvm;
136using namespace llvm::PatternMatch;
137
138namespace {
139//===----------------------------------------------------------------------===//
140// AMDGPU CodeGen Pass Builder interface.
141//===----------------------------------------------------------------------===//
142
143class AMDGPUCodeGenPassBuilder
144 : public CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine> {
145 using Base = CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine>;
146
147public:
148 AMDGPUCodeGenPassBuilder(GCNTargetMachine &TM,
149 const CGPassBuilderOption &Opts,
150 PassInstrumentationCallbacks *PIC);
151
152 void addIRPasses(PassManagerWrapper &PMW) const;
153 void addCodeGenPrepare(PassManagerWrapper &PMW) const;
154 void addPreISel(PassManagerWrapper &PMW) const;
155 void addILPOpts(PassManagerWrapper &PMWM) const;
156 void addAsmPrinterBegin(PassManagerWrapper &PMW) const;
157 void addAsmPrinter(PassManagerWrapper &PMW) const;
158 void addAsmPrinterEnd(PassManagerWrapper &PMW) const;
159 Error addInstSelector(PassManagerWrapper &PMW) const;
160 void addPreRewrite(PassManagerWrapper &PMW) const;
161 void addMachineSSAOptimization(PassManagerWrapper &PMW) const;
162 void addPostRegAlloc(PassManagerWrapper &PMW) const;
163 void addPreEmitPass(PassManagerWrapper &PMWM) const;
164 void addPreEmitRegAlloc(PassManagerWrapper &PMW) const;
165 Error addRegAssignmentFast(PassManagerWrapper &PMW) const;
166 Error addRegAssignmentOptimized(PassManagerWrapper &PMW) const;
167 void addPreRegAlloc(PassManagerWrapper &PMW) const;
168 Error addFastRegAlloc(PassManagerWrapper &PMW) const;
169 Error addOptimizedRegAlloc(PassManagerWrapper &PMW) const;
170 void addPreSched2(PassManagerWrapper &PMW) const;
171 void addPostBBSections(PassManagerWrapper &PMW) const;
172
173private:
174 Error validateRegAllocOptions() const;
175
176public:
177 /// Check if a pass is enabled given \p Opt option. The option always
178 /// overrides defaults if explicitly used. Otherwise its default will be used
179 /// given that a pass shall work at an optimization \p Level minimum.
180 bool isPassEnabled(const cl::opt<bool> &Opt,
181 CodeGenOptLevel Level = CodeGenOptLevel::Default) const;
182 void addEarlyCSEOrGVNPass(PassManagerWrapper &PMW) const;
183 void addStraightLineScalarOptimizationPasses(PassManagerWrapper &PMW) const;
184};
185
186class SGPRRegisterRegAlloc : public RegisterRegAllocBase<SGPRRegisterRegAlloc> {
187public:
188 SGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
189 : RegisterRegAllocBase(N, D, C) {}
190};
191
192class VGPRRegisterRegAlloc : public RegisterRegAllocBase<VGPRRegisterRegAlloc> {
193public:
194 VGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
195 : RegisterRegAllocBase(N, D, C) {}
196};
197
198class WWMRegisterRegAlloc : public RegisterRegAllocBase<WWMRegisterRegAlloc> {
199public:
200 WWMRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
201 : RegisterRegAllocBase(N, D, C) {}
202};
203
204static bool onlyAllocateSGPRs(const TargetRegisterInfo &TRI,
205 const MachineRegisterInfo &MRI,
206 const Register Reg) {
207 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
208 return static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC);
209}
210
211static bool onlyAllocateVGPRs(const TargetRegisterInfo &TRI,
212 const MachineRegisterInfo &MRI,
213 const Register Reg) {
214 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
215 return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC);
216}
217
218static bool onlyAllocateWWMRegs(const TargetRegisterInfo &TRI,
219 const MachineRegisterInfo &MRI,
220 const Register Reg) {
221 const SIMachineFunctionInfo *MFI =
222 MRI.getMF().getInfo<SIMachineFunctionInfo>();
223 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
224 return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC) &&
225 MFI->checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG);
226}
227
228/// -{sgpr|wwm|vgpr}-regalloc=... command line option.
229static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
230
231/// A dummy default pass factory indicates whether the register allocator is
232/// overridden on the command line.
233static llvm::once_flag InitializeDefaultSGPRRegisterAllocatorFlag;
234static llvm::once_flag InitializeDefaultVGPRRegisterAllocatorFlag;
235static llvm::once_flag InitializeDefaultWWMRegisterAllocatorFlag;
236
237static SGPRRegisterRegAlloc
238defaultSGPRRegAlloc("default",
239 "pick SGPR register allocator based on -O option",
240 useDefaultRegisterAllocator);
241
242static cl::opt<SGPRRegisterRegAlloc::FunctionPassCtor, false,
243 RegisterPassParser<SGPRRegisterRegAlloc>>
244SGPRRegAlloc("sgpr-regalloc", cl::Hidden, cl::init(Val: &useDefaultRegisterAllocator),
245 cl::desc("Register allocator to use for SGPRs"));
246
247static cl::opt<VGPRRegisterRegAlloc::FunctionPassCtor, false,
248 RegisterPassParser<VGPRRegisterRegAlloc>>
249VGPRRegAlloc("vgpr-regalloc", cl::Hidden, cl::init(Val: &useDefaultRegisterAllocator),
250 cl::desc("Register allocator to use for VGPRs"));
251
252static cl::opt<WWMRegisterRegAlloc::FunctionPassCtor, false,
253 RegisterPassParser<WWMRegisterRegAlloc>>
254 WWMRegAlloc("wwm-regalloc", cl::Hidden,
255 cl::init(Val: &useDefaultRegisterAllocator),
256 cl::desc("Register allocator to use for WWM registers"));
257
258// New pass manager register allocator options for AMDGPU
259static cl::opt<RegAllocType, false, RegAllocTypeParser> SGPRRegAllocNPM(
260 "sgpr-regalloc-npm", cl::Hidden, cl::init(Val: RegAllocType::Default),
261 cl::desc("Register allocator for SGPRs (new pass manager)"));
262
263static cl::opt<RegAllocType, false, RegAllocTypeParser> VGPRRegAllocNPM(
264 "vgpr-regalloc-npm", cl::Hidden, cl::init(Val: RegAllocType::Default),
265 cl::desc("Register allocator for VGPRs (new pass manager)"));
266
267static cl::opt<RegAllocType, false, RegAllocTypeParser> WWMRegAllocNPM(
268 "wwm-regalloc-npm", cl::Hidden, cl::init(Val: RegAllocType::Default),
269 cl::desc("Register allocator for WWM registers (new pass manager)"));
270
271/// Check if the given RegAllocType is supported for AMDGPU NPM register
272/// allocation. Only Fast and Greedy are supported; Basic and PBQP are not.
273static Error checkRegAllocSupported(RegAllocType RAType, StringRef RegName) {
274 if (RAType == RegAllocType::Basic || RAType == RegAllocType::PBQP) {
275 return make_error<StringError>(
276 Args: Twine("unsupported register allocator '") +
277 (RAType == RegAllocType::Basic ? "basic" : "pbqp") + "' for " +
278 RegName + " registers",
279 Args: inconvertibleErrorCode());
280 }
281 return Error::success();
282}
283
284Error AMDGPUCodeGenPassBuilder::validateRegAllocOptions() const {
285 // 1. Generic --regalloc-npm is not supported for AMDGPU.
286 if (Opt.RegAlloc != RegAllocType::Unset) {
287 return make_error<StringError>(
288 Args: "-regalloc-npm not supported for amdgcn. Use -sgpr-regalloc-npm, "
289 "-vgpr-regalloc-npm, and -wwm-regalloc-npm",
290 Args: inconvertibleErrorCode());
291 }
292
293 // 2. Legacy PM regalloc options are not compatible with NPM.
294 if (SGPRRegAlloc.getNumOccurrences() > 0 ||
295 VGPRRegAlloc.getNumOccurrences() > 0 ||
296 WWMRegAlloc.getNumOccurrences() > 0) {
297 return make_error<StringError>(
298 Args: "-sgpr-regalloc, -vgpr-regalloc, and -wwm-regalloc are legacy PM "
299 "options. Use -sgpr-regalloc-npm, -vgpr-regalloc-npm, and "
300 "-wwm-regalloc-npm with the new pass manager",
301 Args: inconvertibleErrorCode());
302 }
303
304 // 3. Only Fast and Greedy allocators are supported for AMDGPU.
305 if (auto Err = checkRegAllocSupported(RAType: SGPRRegAllocNPM, RegName: "SGPR"))
306 return Err;
307 if (auto Err = checkRegAllocSupported(RAType: WWMRegAllocNPM, RegName: "WWM"))
308 return Err;
309 if (auto Err = checkRegAllocSupported(RAType: VGPRRegAllocNPM, RegName: "VGPR"))
310 return Err;
311
312 return Error::success();
313}
314
315static void initializeDefaultSGPRRegisterAllocatorOnce() {
316 RegisterRegAlloc::FunctionPassCtor Ctor = SGPRRegisterRegAlloc::getDefault();
317
318 if (!Ctor) {
319 Ctor = SGPRRegAlloc;
320 SGPRRegisterRegAlloc::setDefault(SGPRRegAlloc);
321 }
322}
323
324static void initializeDefaultVGPRRegisterAllocatorOnce() {
325 RegisterRegAlloc::FunctionPassCtor Ctor = VGPRRegisterRegAlloc::getDefault();
326
327 if (!Ctor) {
328 Ctor = VGPRRegAlloc;
329 VGPRRegisterRegAlloc::setDefault(VGPRRegAlloc);
330 }
331}
332
333static void initializeDefaultWWMRegisterAllocatorOnce() {
334 RegisterRegAlloc::FunctionPassCtor Ctor = WWMRegisterRegAlloc::getDefault();
335
336 if (!Ctor) {
337 Ctor = WWMRegAlloc;
338 WWMRegisterRegAlloc::setDefault(WWMRegAlloc);
339 }
340}
341
342static FunctionPass *createBasicSGPRRegisterAllocator() {
343 return createBasicRegisterAllocator(F: onlyAllocateSGPRs);
344}
345
346static FunctionPass *createGreedySGPRRegisterAllocator() {
347 return createGreedyRegisterAllocator(F: onlyAllocateSGPRs);
348}
349
350static FunctionPass *createFastSGPRRegisterAllocator() {
351 return createFastRegisterAllocator(F: onlyAllocateSGPRs, ClearVirtRegs: false);
352}
353
354static FunctionPass *createBasicVGPRRegisterAllocator() {
355 return createBasicRegisterAllocator(F: onlyAllocateVGPRs);
356}
357
358static FunctionPass *createGreedyVGPRRegisterAllocator() {
359 return createGreedyRegisterAllocator(F: onlyAllocateVGPRs);
360}
361
362static FunctionPass *createFastVGPRRegisterAllocator() {
363 return createFastRegisterAllocator(F: onlyAllocateVGPRs, ClearVirtRegs: true);
364}
365
366static FunctionPass *createBasicWWMRegisterAllocator() {
367 return createBasicRegisterAllocator(F: onlyAllocateWWMRegs);
368}
369
370static FunctionPass *createGreedyWWMRegisterAllocator() {
371 return createGreedyRegisterAllocator(F: onlyAllocateWWMRegs);
372}
373
374static FunctionPass *createFastWWMRegisterAllocator() {
375 return createFastRegisterAllocator(F: onlyAllocateWWMRegs, ClearVirtRegs: false);
376}
377
378static SGPRRegisterRegAlloc basicRegAllocSGPR(
379 "basic", "basic register allocator", createBasicSGPRRegisterAllocator);
380static SGPRRegisterRegAlloc greedyRegAllocSGPR(
381 "greedy", "greedy register allocator", createGreedySGPRRegisterAllocator);
382
383static SGPRRegisterRegAlloc fastRegAllocSGPR(
384 "fast", "fast register allocator", createFastSGPRRegisterAllocator);
385
386
387static VGPRRegisterRegAlloc basicRegAllocVGPR(
388 "basic", "basic register allocator", createBasicVGPRRegisterAllocator);
389static VGPRRegisterRegAlloc greedyRegAllocVGPR(
390 "greedy", "greedy register allocator", createGreedyVGPRRegisterAllocator);
391
392static VGPRRegisterRegAlloc fastRegAllocVGPR(
393 "fast", "fast register allocator", createFastVGPRRegisterAllocator);
394static WWMRegisterRegAlloc basicRegAllocWWMReg("basic",
395 "basic register allocator",
396 createBasicWWMRegisterAllocator);
397static WWMRegisterRegAlloc
398 greedyRegAllocWWMReg("greedy", "greedy register allocator",
399 createGreedyWWMRegisterAllocator);
400static WWMRegisterRegAlloc fastRegAllocWWMReg("fast", "fast register allocator",
401 createFastWWMRegisterAllocator);
402
403static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {
404 return Phase == ThinOrFullLTOPhase::FullLTOPreLink ||
405 Phase == ThinOrFullLTOPhase::ThinLTOPreLink;
406}
407} // anonymous namespace
408
409static cl::opt<bool>
410EnableEarlyIfConversion("amdgpu-early-ifcvt", cl::Hidden,
411 cl::desc("Run early if-conversion"),
412 cl::init(Val: false));
413
414static cl::opt<bool>
415OptExecMaskPreRA("amdgpu-opt-exec-mask-pre-ra", cl::Hidden,
416 cl::desc("Run pre-RA exec mask optimizations"),
417 cl::init(Val: true));
418
419static cl::opt<bool>
420 LowerCtorDtor("amdgpu-lower-global-ctor-dtor",
421 cl::desc("Lower GPU ctor / dtors to globals on the device."),
422 cl::init(Val: true), cl::Hidden);
423
424// Option to disable vectorizer for tests.
425static cl::opt<bool> EnableLoadStoreVectorizer(
426 "amdgpu-load-store-vectorizer",
427 cl::desc("Enable load store vectorizer"),
428 cl::init(Val: true),
429 cl::Hidden);
430
431// Option to control global loads scalarization
432static cl::opt<bool> ScalarizeGlobal(
433 "amdgpu-scalarize-global-loads",
434 cl::desc("Enable global load scalarization"),
435 cl::init(Val: true),
436 cl::Hidden);
437
438// Option to run internalize pass.
439static cl::opt<bool> InternalizeSymbols(
440 "amdgpu-internalize-symbols",
441 cl::desc("Enable elimination of non-kernel functions and unused globals"),
442 cl::init(Val: false),
443 cl::Hidden);
444
445// Option to inline all early.
446static cl::opt<bool> EarlyInlineAll(
447 "amdgpu-early-inline-all",
448 cl::desc("Inline all functions early"),
449 cl::init(Val: false),
450 cl::Hidden);
451
452static cl::opt<bool> RemoveIncompatibleFunctions(
453 "amdgpu-enable-remove-incompatible-functions", cl::Hidden,
454 cl::desc("Enable removal of functions when they"
455 "use features not supported by the target GPU"),
456 cl::init(Val: true));
457
458static cl::opt<bool> EnableSDWAPeephole(
459 "amdgpu-sdwa-peephole",
460 cl::desc("Enable SDWA peepholer"),
461 cl::init(Val: true));
462
463static cl::opt<bool> EnableDPPCombine(
464 "amdgpu-dpp-combine",
465 cl::desc("Enable DPP combiner"),
466 cl::init(Val: true));
467
468// Enable address space based alias analysis
469static cl::opt<bool> EnableAMDGPUAliasAnalysis("enable-amdgpu-aa", cl::Hidden,
470 cl::desc("Enable AMDGPU Alias Analysis"),
471 cl::init(Val: true));
472
473// Enable lib calls simplifications
474static cl::opt<bool> EnableLibCallSimplify(
475 "amdgpu-simplify-libcall",
476 cl::desc("Enable amdgpu library simplifications"),
477 cl::init(Val: true),
478 cl::Hidden);
479
480static cl::opt<bool> EnableLowerKernelArguments(
481 "amdgpu-ir-lower-kernel-arguments",
482 cl::desc("Lower kernel argument loads in IR pass"),
483 cl::init(Val: true),
484 cl::Hidden);
485
486static cl::opt<bool> EnableRegReassign(
487 "amdgpu-reassign-regs",
488 cl::desc("Enable register reassign optimizations on gfx10+"),
489 cl::init(Val: true),
490 cl::Hidden);
491
492static cl::opt<bool> OptVGPRLiveRange(
493 "amdgpu-opt-vgpr-liverange",
494 cl::desc("Enable VGPR liverange optimizations for if-else structure"),
495 cl::init(Val: true), cl::Hidden);
496
497static cl::opt<ScanOptions> AMDGPUAtomicOptimizerStrategy(
498 "amdgpu-atomic-optimizer-strategy",
499 cl::desc("Select DPP or Iterative strategy for scan"),
500 cl::init(Val: ScanOptions::Iterative),
501 cl::values(
502 clEnumValN(ScanOptions::DPP, "DPP", "Use DPP operations for scan"),
503 clEnumValN(ScanOptions::Iterative, "Iterative",
504 "Use Iterative approach for scan"),
505 clEnumValN(ScanOptions::None, "None", "Disable atomic optimizer")));
506
507// Enable Mode register optimization
508static cl::opt<bool> EnableSIModeRegisterPass(
509 "amdgpu-mode-register",
510 cl::desc("Enable mode register pass"),
511 cl::init(Val: true),
512 cl::Hidden);
513
514// Enable GFX11+ s_delay_alu insertion
515static cl::opt<bool>
516 EnableInsertDelayAlu("amdgpu-enable-delay-alu",
517 cl::desc("Enable s_delay_alu insertion"),
518 cl::init(Val: true), cl::Hidden);
519
520// Enable GFX11+ VOPD
521static cl::opt<bool>
522 EnableVOPD("amdgpu-enable-vopd",
523 cl::desc("Enable VOPD, dual issue of VALU in wave32"),
524 cl::init(Val: true), cl::Hidden);
525
526// Option is used in lit tests to prevent deadcoding of patterns inspected.
527static cl::opt<bool>
528EnableDCEInRA("amdgpu-dce-in-ra",
529 cl::init(Val: true), cl::Hidden,
530 cl::desc("Enable machine DCE inside regalloc"));
531
532static cl::opt<bool> EnableSetWavePriority("amdgpu-set-wave-priority",
533 cl::desc("Adjust wave priority"),
534 cl::init(Val: false), cl::Hidden);
535
536static cl::opt<bool> EnableScalarIRPasses(
537 "amdgpu-scalar-ir-passes",
538 cl::desc("Enable scalar IR passes"),
539 cl::init(Val: true),
540 cl::Hidden);
541
542static cl::opt<bool> EnableLowerExecSync(
543 "amdgpu-enable-lower-exec-sync",
544 cl::desc("Enable lowering of execution synchronization."), cl::init(Val: true),
545 cl::Hidden);
546
547static cl::opt<bool>
548 EnableSwLowerLDS("amdgpu-enable-sw-lower-lds",
549 cl::desc("Enable lowering of lds to global memory pass "
550 "and asan instrument resulting IR."),
551 cl::init(Val: true), cl::Hidden);
552
553static cl::opt<bool, true> EnableObjectLinking(
554 "amdgpu-enable-object-linking",
555 cl::desc("Enable object linking for cross-TU LDS and ABI support"),
556 cl::location(L&: AMDGPUTargetMachine::EnableObjectLinking), cl::init(Val: false),
557 cl::Hidden);
558
559static cl::opt<bool, true> EnableLowerModuleLDS(
560 "amdgpu-enable-lower-module-lds", cl::desc("Enable lower module lds pass"),
561 cl::location(L&: AMDGPUTargetMachine::EnableLowerModuleLDS), cl::init(Val: true),
562 cl::Hidden);
563
564static cl::opt<bool> EnablePreRAOptimizations(
565 "amdgpu-enable-pre-ra-optimizations",
566 cl::desc("Enable Pre-RA optimizations pass"), cl::init(Val: true),
567 cl::Hidden);
568
569static cl::opt<bool> EnablePromoteKernelArguments(
570 "amdgpu-enable-promote-kernel-arguments",
571 cl::desc("Enable promotion of flat kernel pointer arguments to global"),
572 cl::Hidden, cl::init(Val: true));
573
574static cl::opt<bool> EnableImageIntrinsicOptimizer(
575 "amdgpu-enable-image-intrinsic-optimizer",
576 cl::desc("Enable image intrinsic optimizer pass"), cl::init(Val: true),
577 cl::Hidden);
578
579static cl::opt<bool>
580 EnableLoopPrefetch("amdgpu-loop-prefetch",
581 cl::desc("Enable loop data prefetch on AMDGPU"),
582 cl::Hidden, cl::init(Val: false));
583
584static cl::opt<std::string>
585 AMDGPUSchedStrategy("amdgpu-sched-strategy",
586 cl::desc("Select custom AMDGPU scheduling strategy."),
587 cl::Hidden, cl::init(Val: ""));
588
589// Scheduler selection is consulted both when creating the scheduler and from
590// overrideSchedPolicy(), so keep the attribute and global command line handling
591// in one helper.
592StringRef llvm::AMDGPU::getSchedStrategy(const Function &F) {
593 Attribute SchedStrategyAttr = F.getFnAttribute(Kind: "amdgpu-sched-strategy");
594 if (SchedStrategyAttr.isValid())
595 return SchedStrategyAttr.getValueAsString();
596
597 if (!AMDGPUSchedStrategy.empty())
598 return AMDGPUSchedStrategy;
599
600 return "";
601}
602
603static void
604diagnoseUnsupportedCoExecSchedulerSelection(const Function &F,
605 const GCNSubtarget &ST) {
606 if (ST.hasGFX1250Insts())
607 return;
608
609 F.getContext().diagnose(DI: DiagnosticInfoUnsupported(
610 F, "'amdgpu-sched-strategy'='coexec' is only supported for gfx1250",
611 DiagnosticLocation(), DS_Warning));
612}
613
614static bool useNoopPostScheduler(const Function &F) {
615 Attribute PostSchedStrategyAttr =
616 F.getFnAttribute(Kind: "amdgpu-post-sched-strategy");
617 return PostSchedStrategyAttr.isValid() &&
618 PostSchedStrategyAttr.getValueAsString() == "nop";
619}
620
621static cl::opt<bool> EnableRewritePartialRegUses(
622 "amdgpu-enable-rewrite-partial-reg-uses",
623 cl::desc("Enable rewrite partial reg uses pass"), cl::init(Val: true),
624 cl::Hidden);
625
626static cl::opt<bool> EnableHipStdPar(
627 "amdgpu-enable-hipstdpar",
628 cl::desc("Enable HIP Standard Parallelism Offload support"), cl::init(Val: false),
629 cl::Hidden);
630
631static cl::opt<bool>
632 EnableAMDGPUAttributor("amdgpu-attributor-enable",
633 cl::desc("Enable AMDGPUAttributorPass"),
634 cl::init(Val: true), cl::Hidden);
635
636static cl::opt<bool> HasClosedWorldAssumption(
637 "amdgpu-link-time-closed-world",
638 cl::desc("Whether has closed-world assumption at link time"),
639 cl::init(Val: false), cl::Hidden);
640
641static cl::opt<bool> EnableUniformIntrinsicCombine(
642 "amdgpu-enable-uniform-intrinsic-combine",
643 cl::desc("Enable/Disable the Uniform Intrinsic Combine Pass"),
644 cl::init(Val: true), cl::Hidden);
645
646extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
647 // Register the target
648 RegisterTargetMachine<R600TargetMachine> X(getTheR600Target());
649 RegisterTargetMachine<GCNTargetMachine> Y(getTheGCNTarget());
650 RegisterTargetMachine<GCNTargetMachine> YLegacy(getTheGCNLegacyTarget());
651
652 PassRegistry *PR = PassRegistry::getPassRegistry();
653 initializeR600ClauseMergePassPass(*PR);
654 initializeR600ControlFlowFinalizerPass(*PR);
655 initializeR600PacketizerPass(*PR);
656 initializeR600ExpandSpecialInstrsPassPass(*PR);
657 initializeR600VectorRegMergerPass(*PR);
658 initializeR600EmitClauseMarkersPass(*PR);
659 initializeR600MachineCFGStructurizerPass(*PR);
660 initializeGlobalISel(*PR);
661 initializeAMDGPUAsmPrinterPass(*PR);
662 initializeAMDGPUDAGToDAGISelLegacyPass(*PR);
663 initializeAMDGPUPrepareAGPRAllocLegacyPass(*PR);
664 initializeGCNDPPCombineLegacyPass(*PR);
665 initializeSILowerI1CopiesLegacyPass(*PR);
666 initializeAMDGPUGlobalISelDivergenceLoweringPass(*PR);
667 initializeAMDGPURegBankSelectPass(*PR);
668 initializeAMDGPURegBankLegalizePass(*PR);
669 initializeSILowerWWMCopiesLegacyPass(*PR);
670 initializeAMDGPUMarkLastScratchLoadLegacyPass(*PR);
671 initializeSILowerSGPRSpillsLegacyPass(*PR);
672 initializeSIFixSGPRCopiesLegacyPass(*PR);
673 initializeSIFixVGPRCopiesLegacyPass(*PR);
674 initializeSIFoldOperandsLegacyPass(*PR);
675 initializeSIPeepholeSDWALegacyPass(*PR);
676 initializeSIShrinkInstructionsLegacyPass(*PR);
677 initializeSIOptimizeExecMaskingPreRALegacyPass(*PR);
678 initializeSIOptimizeVGPRLiveRangeLegacyPass(*PR);
679 initializeAMDGPUNextUseAnalysisLegacyPassPass(*PR);
680 initializeAMDGPUNextUseAnalysisPrinterLegacyPassPass(*PR);
681 initializeSILoadStoreOptimizerLegacyPass(*PR);
682 initializeAMDGPUCtorDtorLoweringLegacyPass(*PR);
683 initializeAMDGPUAlwaysInlinePass(*PR);
684 initializeAMDGPULowerExecSyncLegacyPass(*PR);
685 initializeAMDGPUSwLowerLDSLegacyPass(*PR);
686 initializeAMDGPUAnnotateUniformValuesLegacyPass(*PR);
687 initializeAMDGPUAtomicOptimizerPass(*PR);
688 initializeAMDGPULowerKernelArgumentsPass(*PR);
689 initializeAMDGPUPromoteKernelArgumentsPass(*PR);
690 initializeAMDGPULowerKernelAttributesPass(*PR);
691 initializeAMDGPUExportKernelRuntimeHandlesLegacyPass(*PR);
692 initializeAMDGPUPostLegalizerCombinerPass(*PR);
693 initializeAMDGPUPreLegalizerCombinerPass(*PR);
694 initializeAMDGPURegBankCombinerPass(*PR);
695 initializeAMDGPUPromoteAllocaPass(*PR);
696 initializeAMDGPUCodeGenPreparePass(*PR);
697 initializeAMDGPULateCodeGenPrepareLegacyPass(*PR);
698 initializeAMDGPURemoveIncompatibleFunctionsLegacyPass(*PR);
699 initializeAMDGPULowerModuleLDSLegacyPass(*PR);
700 initializeAMDGPULowerBufferFatPointersPass(*PR);
701 initializeAMDGPULowerIntrinsicsLegacyPass(*PR);
702 initializeAMDGPUReserveWWMRegsLegacyPass(*PR);
703 initializeAMDGPURewriteAGPRCopyMFMALegacyPass(*PR);
704 initializeAMDGPURewriteOutArgumentsPass(*PR);
705 initializeAMDGPURewriteUndefForPHILegacyPass(*PR);
706 initializeSIAnnotateControlFlowLegacyPass(*PR);
707 initializeAMDGPUInsertDelayAluLegacyPass(*PR);
708 initializeAMDGPULowerVGPREncodingLegacyPass(*PR);
709 initializeSIInsertHardClausesLegacyPass(*PR);
710 initializeSIInsertWaitcntsLegacyPass(*PR);
711 initializeSIModeRegisterLegacyPass(*PR);
712 initializeSIWholeQuadModeLegacyPass(*PR);
713 initializeSILowerControlFlowLegacyPass(*PR);
714 initializeSIPreEmitPeepholeLegacyPass(*PR);
715 initializeSILateBranchLoweringLegacyPass(*PR);
716 initializeSIMemoryLegalizerLegacyPass(*PR);
717 initializeSIOptimizeExecMaskingLegacyPass(*PR);
718 initializeSIPreAllocateWWMRegsLegacyPass(*PR);
719 initializeSIFormMemoryClausesLegacyPass(*PR);
720 initializeSIPostRABundlerLegacyPass(*PR);
721 initializeGCNCreateVOPDLegacyPass(*PR);
722 initializeAMDGPUUnifyDivergentExitNodesLegacyPass(*PR);
723 initializeAMDGPUAAWrapperPassPass(*PR);
724 initializeAMDGPUExternalAAWrapperPass(*PR);
725 initializeAMDGPUImageIntrinsicOptimizerPass(*PR);
726 initializeAMDGPUPrintfRuntimeBindingPass(*PR);
727 initializeAMDGPUResourceUsageAnalysisWrapperPassPass(*PR);
728 initializeGCNNSAReassignLegacyPass(*PR);
729 initializeGCNPreRAOptimizationsLegacyPass(*PR);
730 initializeGCNPreRALongBranchRegLegacyPass(*PR);
731 initializeGCNRewritePartialRegUsesLegacyPass(*PR);
732 initializeGCNRegPressurePrinterPass(*PR);
733 initializeAMDGPUPreloadKernArgPrologLegacyPass(*PR);
734 initializeAMDGPUWaitSGPRHazardsLegacyPass(*PR);
735 initializeAMDGPUPreloadKernelArgumentsLegacyPass(*PR);
736 initializeAMDGPUUniformIntrinsicCombineLegacyPass(*PR);
737}
738
739static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
740 return std::make_unique<AMDGPUTargetObjectFile>();
741}
742
743static ScheduleDAGInstrs *createSIMachineScheduler(MachineSchedContext *C) {
744 return new SIScheduleDAGMI(C);
745}
746
747static ScheduleDAGInstrs *
748createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
749 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
750 ScheduleDAGMILive *DAG =
751 new GCNScheduleDAGMILive(C, std::make_unique<GCNMaxOccupancySchedStrategy>(args&: C));
752 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
753 if (ST.shouldClusterStores())
754 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
755 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
756 DAG->addMutation(Mutation: createAMDGPUMacroFusionDAGMutation());
757 DAG->addMutation(Mutation: createAMDGPUExportClusteringDAGMutation());
758 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
759 DAG->addMutation(Mutation: createAMDGPUHazardLatencyDAGMutation(MF: C->MF));
760 return DAG;
761}
762
763static ScheduleDAGInstrs *
764createGCNMaxILPMachineScheduler(MachineSchedContext *C) {
765 ScheduleDAGMILive *DAG =
766 new GCNScheduleDAGMILive(C, std::make_unique<GCNMaxILPSchedStrategy>(args&: C));
767 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
768 return DAG;
769}
770
771static ScheduleDAGInstrs *
772createGCNMaxMemoryClauseMachineScheduler(MachineSchedContext *C) {
773 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
774 ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive(
775 C, std::make_unique<GCNMaxMemoryClauseSchedStrategy>(args&: C));
776 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
777 if (ST.shouldClusterStores())
778 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
779 DAG->addMutation(Mutation: createAMDGPUExportClusteringDAGMutation());
780 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
781 DAG->addMutation(Mutation: createAMDGPUHazardLatencyDAGMutation(MF: C->MF));
782 return DAG;
783}
784
785static ScheduleDAGInstrs *
786createIterativeGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
787 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
788 auto *DAG = new GCNIterativeScheduler(
789 C, GCNIterativeScheduler::SCHEDULE_LEGACYMAXOCCUPANCY);
790 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
791 if (ST.shouldClusterStores())
792 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
793 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
794 return DAG;
795}
796
797static ScheduleDAGInstrs *createMinRegScheduler(MachineSchedContext *C) {
798 auto *DAG = new GCNIterativeScheduler(
799 C, GCNIterativeScheduler::SCHEDULE_MINREGFORCED);
800 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
801 return DAG;
802}
803
804static ScheduleDAGInstrs *
805createIterativeILPMachineScheduler(MachineSchedContext *C) {
806 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
807 auto *DAG = new GCNIterativeScheduler(C, GCNIterativeScheduler::SCHEDULE_ILP);
808 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
809 if (ST.shouldClusterStores())
810 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
811 DAG->addMutation(Mutation: createAMDGPUMacroFusionDAGMutation());
812 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
813 return DAG;
814}
815
816static MachineSchedRegistry
817SISchedRegistry("si", "Run SI's custom scheduler",
818 createSIMachineScheduler);
819
820static MachineSchedRegistry
821GCNMaxOccupancySchedRegistry("gcn-max-occupancy",
822 "Run GCN scheduler to maximize occupancy",
823 createGCNMaxOccupancyMachineScheduler);
824
825static MachineSchedRegistry
826 GCNMaxILPSchedRegistry("gcn-max-ilp", "Run GCN scheduler to maximize ilp",
827 createGCNMaxILPMachineScheduler);
828
829static MachineSchedRegistry GCNMaxMemoryClauseSchedRegistry(
830 "gcn-max-memory-clause", "Run GCN scheduler to maximize memory clause",
831 createGCNMaxMemoryClauseMachineScheduler);
832
833static MachineSchedRegistry IterativeGCNMaxOccupancySchedRegistry(
834 "gcn-iterative-max-occupancy-experimental",
835 "Run GCN scheduler to maximize occupancy (experimental)",
836 createIterativeGCNMaxOccupancyMachineScheduler);
837
838static MachineSchedRegistry GCNMinRegSchedRegistry(
839 "gcn-iterative-minreg",
840 "Run GCN iterative scheduler for minimal register usage (experimental)",
841 createMinRegScheduler);
842
843static MachineSchedRegistry GCNILPSchedRegistry(
844 "gcn-iterative-ilp",
845 "Run GCN iterative scheduler for ILP scheduling (experimental)",
846 createIterativeILPMachineScheduler);
847
848LLVM_READNONE
849static StringRef getGPUOrDefault(const Triple &TT, StringRef GPU) {
850 if (!GPU.empty())
851 return GPU;
852
853 if (StringRef Name = AMDGPU::getArchNameFromSubArch(SubArch: TT.getSubArch());
854 !Name.empty())
855 return Name;
856
857 // Need to default to a target with flat support for HSA.
858 if (TT.isAMDGCN())
859 return TT.getOS() == Triple::AMDHSA ? "generic-hsa" : "generic";
860
861 return "r600";
862}
863
864static Reloc::Model getEffectiveRelocModel() {
865 // The AMDGPU toolchain only supports generating shared objects, so we
866 // must always use PIC.
867 return Reloc::PIC_;
868}
869
870AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT,
871 StringRef CPU, StringRef FS,
872 const TargetOptions &Options,
873 std::optional<Reloc::Model> RM,
874 std::optional<CodeModel::Model> CM,
875 CodeGenOptLevel OptLevel)
876 : CodeGenTargetMachineImpl(
877 T, TT.computeDataLayout(), TT, getGPUOrDefault(TT, GPU: CPU), FS, Options,
878 getEffectiveRelocModel(), getEffectiveCodeModel(CM, Default: CodeModel::Small),
879 OptLevel),
880 TLOF(createTLOF(TT: getTargetTriple())) {
881 initAsmInfo();
882 if (TT.isAMDGCN()) {
883 // Triple is missing a representation for non-empty, but unrecognized
884 // subarches. Only permit no subarch for any subtarget if it was really
885 // empty.
886 bool IsUnknownSubArch =
887 TT.getSubArch() == Triple::NoSubArch && TT.getArchName().size() != 6;
888 if (IsUnknownSubArch)
889 reportFatalUsageError(reason: "unknown subarch " + TT.getArchName());
890
891 if (TT.getSubArch() != Triple::NoSubArch) {
892 AMDGPU::GPUKind Kind = AMDGPU::parseArchAMDGCN(CPU);
893 Triple::SubArchType GPUSubArch = AMDGPU::getSubArch(AK: Kind);
894 if (Kind != AMDGPU::GK_NONE && GPUSubArch != TT.getSubArch() &&
895 TT.getSubArch() != AMDGPU::getMajorSubArch(SubArch: GPUSubArch)) {
896 reportFatalUsageError(reason: "invalid cpu '" + CPU + "' for subarch " +
897 TT.getArchName());
898 }
899 }
900
901 if (getMCSubtargetInfo().checkFeatures(FS: "+wavefrontsize64"))
902 MRI.reset(p: llvm::createGCNMCRegisterInfo(DwarfFlavour: AMDGPUDwarfFlavour::Wave64));
903 else if (getMCSubtargetInfo().checkFeatures(FS: "+wavefrontsize32"))
904 MRI.reset(p: llvm::createGCNMCRegisterInfo(DwarfFlavour: AMDGPUDwarfFlavour::Wave32));
905 }
906 LLT::setUseExtended(true);
907}
908
909bool AMDGPUTargetMachine::EnableFunctionCalls = false;
910bool AMDGPUTargetMachine::EnableObjectLinking = false;
911bool AMDGPUTargetMachine::EnableLowerModuleLDS = true;
912
913AMDGPUTargetMachine::~AMDGPUTargetMachine() = default;
914
915StringRef AMDGPUTargetMachine::getGPUName(const Function &F) const {
916 Attribute GPUAttr = F.getFnAttribute(Kind: "target-cpu");
917 return GPUAttr.isValid() ? GPUAttr.getValueAsString() : getTargetCPU();
918}
919
920StringRef AMDGPUTargetMachine::getFeatureString(const Function &F) const {
921 Attribute FSAttr = F.getFnAttribute(Kind: "target-features");
922
923 return FSAttr.isValid() ? FSAttr.getValueAsString()
924 : getTargetFeatureString();
925}
926
927llvm::ScheduleDAGInstrs *
928AMDGPUTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
929 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
930 ScheduleDAGMILive *DAG = createSchedLive(C);
931 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
932 if (ST.shouldClusterStores())
933 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
934 return DAG;
935}
936
937/// Predicate for Internalize pass.
938static bool mustPreserveGV(const GlobalValue &GV) {
939 if (const Function *F = dyn_cast<Function>(Val: &GV))
940 return F->isDeclaration() || F->getName().starts_with(Prefix: "__asan_") ||
941 F->getName().starts_with(Prefix: "__sanitizer_") ||
942 AMDGPU::isEntryFunctionCC(CC: F->getCallingConv());
943
944 GV.removeDeadConstantUsers();
945 return !GV.use_empty();
946}
947
948void AMDGPUTargetMachine::registerDefaultAliasAnalyses(AAManager &AAM) {
949 if (EnableAMDGPUAliasAnalysis)
950 AAM.registerFunctionAnalysis<AMDGPUAA>();
951}
952
953static Expected<ScanOptions>
954parseAMDGPUAtomicOptimizerStrategy(StringRef Params) {
955 if (Params.empty())
956 return ScanOptions::Iterative;
957 Params.consume_front(Prefix: "strategy=");
958 auto Result = StringSwitch<std::optional<ScanOptions>>(Params)
959 .Case(S: "dpp", Value: ScanOptions::DPP)
960 .Cases(CaseStrings: {"iterative", ""}, Value: ScanOptions::Iterative)
961 .Case(S: "none", Value: ScanOptions::None)
962 .Default(Value: std::nullopt);
963 if (Result)
964 return *Result;
965 return make_error<StringError>(Args: "invalid parameter", Args: inconvertibleErrorCode());
966}
967
968Expected<AMDGPUAttributorOptions>
969parseAMDGPUAttributorPassOptions(StringRef Params) {
970 AMDGPUAttributorOptions Result;
971 while (!Params.empty()) {
972 StringRef ParamName;
973 std::tie(args&: ParamName, args&: Params) = Params.split(Separator: ';');
974 if (ParamName == "closed-world") {
975 Result.IsClosedWorld = true;
976 } else {
977 return make_error<StringError>(
978 Args: formatv(Fmt: "invalid AMDGPUAttributor pass parameter '{0}' ", Vals&: ParamName)
979 .str(),
980 Args: inconvertibleErrorCode());
981 }
982 }
983 return Result;
984}
985
986void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
987
988#define GET_PASS_REGISTRY "AMDGPUPassRegistry.def"
989#include "llvm/Passes/TargetPassRegistry.inc"
990
991 // TODO: Move this into the base CodeGenPassBuilder once all
992 // targets that currently implement it have a ported asm-printer pass.
993 if (PIC) {
994 PIC->addClassToPassName(ClassName: AMDGPUAsmPrinterBeginPass::name(),
995 PassName: "amdgpu-asm-printer-begin");
996 PIC->addClassToPassName(ClassName: AMDGPUAsmPrinterPass::name(), PassName: "amdgpu-asm-printer");
997 PIC->addClassToPassName(ClassName: AMDGPUAsmPrinterEndPass::name(),
998 PassName: "amdgpu-asm-printer-end");
999 }
1000
1001 PB.registerPipelineParsingCallback(
1002 C: [this](StringRef Name, CGSCCPassManager &PM,
1003 ArrayRef<PassBuilder::PipelineElement> Pipeline) {
1004 if (Name == "amdgpu-attributor-cgscc" && getTargetTriple().isAMDGCN()) {
1005 PM.addPass(Pass: AMDGPUAttributorCGSCCPass(
1006 *static_cast<GCNTargetMachine *>(this)));
1007 return true;
1008 }
1009 return false;
1010 });
1011
1012 PB.registerScalarOptimizerLateEPCallback(
1013 C: [](FunctionPassManager &FPM, OptimizationLevel Level) {
1014 if (Level == OptimizationLevel::O0)
1015 return;
1016
1017 FPM.addPass(Pass: InferAddressSpacesPass());
1018 });
1019
1020 PB.registerVectorizerEndEPCallback(
1021 C: [](FunctionPassManager &FPM, OptimizationLevel Level) {
1022 if (Level == OptimizationLevel::O0)
1023 return;
1024
1025 FPM.addPass(Pass: InferAddressSpacesPass());
1026 });
1027
1028 PB.registerPipelineEarlySimplificationEPCallback(
1029 C: [this](ModulePassManager &PM, OptimizationLevel Level,
1030 ThinOrFullLTOPhase Phase) {
1031 if (!isLTOPreLink(Phase) && getTargetTriple().isAMDGCN()) {
1032 // When we are not using -fgpu-rdc, we can run accelerator code
1033 // selection relatively early, but still after linking to prevent
1034 // eager removal of potentially reachable symbols.
1035 if (EnableHipStdPar) {
1036 PM.addPass(Pass: HipStdParMathFixupPass());
1037 PM.addPass(Pass: HipStdParAcceleratorCodeSelectionPass());
1038 }
1039
1040 PM.addPass(Pass: AMDGPUPrintfRuntimeBindingPass());
1041 }
1042
1043 if (Level == OptimizationLevel::O0)
1044 return;
1045
1046 // We don't want to run internalization at per-module stage.
1047 if (InternalizeSymbols && !isLTOPreLink(Phase)) {
1048 PM.addPass(Pass: InternalizePass(mustPreserveGV));
1049 PM.addPass(Pass: GlobalDCEPass());
1050 }
1051
1052 if (EarlyInlineAll && !EnableFunctionCalls)
1053 PM.addPass(Pass: AMDGPUAlwaysInlinePass());
1054 });
1055
1056 PB.registerPeepholeEPCallback(
1057 C: [](FunctionPassManager &FPM, OptimizationLevel Level) {
1058 if (Level == OptimizationLevel::O0)
1059 return;
1060
1061 FPM.addPass(Pass: AMDGPUUseNativeCallsPass());
1062 if (EnableLibCallSimplify)
1063 FPM.addPass(Pass: AMDGPUSimplifyLibCallsPass());
1064
1065 if (EnableUniformIntrinsicCombine)
1066 FPM.addPass(Pass: AMDGPUUniformIntrinsicCombinePass());
1067 });
1068
1069 PB.registerCGSCCOptimizerLateEPCallback(
1070 C: [this](CGSCCPassManager &PM, OptimizationLevel Level) {
1071 if (Level == OptimizationLevel::O0)
1072 return;
1073
1074 FunctionPassManager FPM;
1075
1076 // Add promote kernel arguments pass to the opt pipeline right before
1077 // infer address spaces which is needed to do actual address space
1078 // rewriting.
1079 if (Level > OptimizationLevel::O1 && EnablePromoteKernelArguments)
1080 FPM.addPass(Pass: AMDGPUPromoteKernelArgumentsPass());
1081
1082 // Add infer address spaces pass to the opt pipeline after inlining
1083 // but before SROA to increase SROA opportunities.
1084 FPM.addPass(Pass: InferAddressSpacesPass());
1085
1086 // This should run after inlining to have any chance of doing
1087 // anything, and before other cleanup optimizations.
1088 FPM.addPass(Pass: AMDGPULowerKernelAttributesPass());
1089
1090 // Promote alloca to vector before SROA and loop unroll. If we
1091 // manage to eliminate allocas before unroll we may choose to unroll
1092 // less.
1093 FPM.addPass(Pass: AMDGPUPromoteAllocaToVectorPass(*this));
1094
1095 PM.addPass(Pass: createCGSCCToFunctionPassAdaptor(Pass: std::move(FPM)));
1096 });
1097
1098 // FIXME: Why is AMDGPUAttributor not in CGSCC?
1099 PB.registerOptimizerLastEPCallback(C: [this](ModulePassManager &MPM,
1100 OptimizationLevel Level,
1101 ThinOrFullLTOPhase Phase) {
1102 if (Level != OptimizationLevel::O0) {
1103 if (!isLTOPreLink(Phase)) {
1104 if (EnableAMDGPUAttributor && getTargetTriple().isAMDGCN()) {
1105 AMDGPUAttributorOptions Opts;
1106 MPM.addPass(Pass: AMDGPUAttributorPass(*this, Opts, Phase));
1107 }
1108 }
1109 }
1110 });
1111
1112 PB.registerFullLinkTimeOptimizationLastEPCallback(
1113 C: [this](ModulePassManager &PM, OptimizationLevel Level) {
1114 // Clean up redundant memory round-trips that the full-LTO pipeline,
1115 // unlike the non-LTO/ThinLTO ones, otherwise leaves for codegen.
1116 if (Level != OptimizationLevel::O0) {
1117 PM.addPass(Pass: createModuleToFunctionPassAdaptor(
1118 Pass: EarlyCSEPass(/*UseMemorySSA=*/true)));
1119 }
1120
1121 // When we are using -fgpu-rdc, we can only run accelerator code
1122 // selection after linking to prevent, otherwise we end up removing
1123 // potentially reachable symbols that were exported as external in other
1124 // modules.
1125 if (EnableHipStdPar) {
1126 PM.addPass(Pass: HipStdParMathFixupPass());
1127 PM.addPass(Pass: HipStdParAcceleratorCodeSelectionPass());
1128 }
1129 // We want to support the -lto-partitions=N option as "best effort".
1130 // For that, we need to lower LDS earlier in the pipeline before the
1131 // module is partitioned for codegen.
1132 if (EnableLowerExecSync)
1133 PM.addPass(Pass: AMDGPULowerExecSyncPass());
1134 if (EnableSwLowerLDS)
1135 PM.addPass(Pass: AMDGPUSwLowerLDSPass());
1136 if (EnableLowerModuleLDS)
1137 PM.addPass(Pass: AMDGPULowerModuleLDSPass(*this));
1138 if (Level != OptimizationLevel::O0) {
1139 // We only want to run this with O2 or higher since inliner and SROA
1140 // don't run in O1.
1141 if (Level != OptimizationLevel::O1) {
1142 PM.addPass(
1143 Pass: createModuleToFunctionPassAdaptor(Pass: InferAddressSpacesPass()));
1144 }
1145 // Do we really need internalization in LTO?
1146 if (InternalizeSymbols) {
1147 PM.addPass(Pass: InternalizePass(mustPreserveGV));
1148 PM.addPass(Pass: GlobalDCEPass());
1149 }
1150 if (EnableAMDGPUAttributor && getTargetTriple().isAMDGCN()) {
1151 AMDGPUAttributorOptions Opt;
1152 if (HasClosedWorldAssumption)
1153 Opt.IsClosedWorld = true;
1154 PM.addPass(Pass: AMDGPUAttributorPass(
1155 *this, Opt, ThinOrFullLTOPhase::FullLTOPostLink));
1156 }
1157 }
1158 if (!NoKernelInfoEndLTO) {
1159 FunctionPassManager FPM;
1160 FPM.addPass(Pass: KernelInfoPrinter(this));
1161 PM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
1162 }
1163 });
1164
1165 PB.registerRegClassFilterParsingCallback(
1166 C: [](StringRef FilterName) -> RegAllocFilterFunc {
1167 if (FilterName == "sgpr")
1168 return onlyAllocateSGPRs;
1169 if (FilterName == "vgpr")
1170 return onlyAllocateVGPRs;
1171 if (FilterName == "wwm")
1172 return onlyAllocateWWMRegs;
1173 return nullptr;
1174 });
1175}
1176
1177bool AMDGPUTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
1178 unsigned DestAS) const {
1179 return AMDGPU::isFlatGlobalAddrSpace(AS: SrcAS) &&
1180 AMDGPU::isFlatGlobalAddrSpace(AS: DestAS);
1181}
1182
1183unsigned AMDGPUTargetMachine::getAssumedAddrSpace(const Value *V) const {
1184 if (auto *Arg = dyn_cast<Argument>(Val: V);
1185 Arg &&
1186 AMDGPU::isModuleEntryFunctionCC(CC: Arg->getParent()->getCallingConv()) &&
1187 !Arg->hasByRefAttr())
1188 return AMDGPUAS::GLOBAL_ADDRESS;
1189
1190 const auto *LD = dyn_cast<LoadInst>(Val: V);
1191 if (!LD) // TODO: Handle invariant load like constant.
1192 return AMDGPUAS::UNKNOWN_ADDRESS_SPACE;
1193
1194 // It must be a generic pointer loaded.
1195 assert(V->getType()->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS);
1196
1197 const auto *Ptr = LD->getPointerOperand();
1198 if (Ptr->getType()->getPointerAddressSpace() != AMDGPUAS::CONSTANT_ADDRESS)
1199 return AMDGPUAS::UNKNOWN_ADDRESS_SPACE;
1200 // For a generic pointer loaded from the constant memory, it could be assumed
1201 // as a global pointer since the constant memory is only populated on the
1202 // host side. As implied by the offload programming model, only global
1203 // pointers could be referenced on the host side.
1204 return AMDGPUAS::GLOBAL_ADDRESS;
1205}
1206
1207std::pair<const Value *, unsigned>
1208AMDGPUTargetMachine::getPredicatedAddrSpace(const Value *V) const {
1209 if (auto *II = dyn_cast<IntrinsicInst>(Val: V)) {
1210 switch (II->getIntrinsicID()) {
1211 case Intrinsic::amdgcn_is_shared:
1212 return std::pair(II->getArgOperand(i: 0), AMDGPUAS::LOCAL_ADDRESS);
1213 case Intrinsic::amdgcn_is_private:
1214 return std::pair(II->getArgOperand(i: 0), AMDGPUAS::PRIVATE_ADDRESS);
1215 default:
1216 break;
1217 }
1218 return std::pair(nullptr, -1);
1219 }
1220 // Check the global pointer predication based on
1221 // (!is_share(p) && !is_private(p)). Note that logic 'and' is commutative and
1222 // the order of 'is_shared' and 'is_private' is not significant.
1223 Value *Ptr;
1224 if (match(
1225 V: const_cast<Value *>(V),
1226 P: m_c_And(L: m_Not(V: m_Intrinsic<Intrinsic::amdgcn_is_shared>(Ops: m_Value(V&: Ptr))),
1227 R: m_Not(V: m_Intrinsic<Intrinsic::amdgcn_is_private>(
1228 Ops: m_Deferred(V: Ptr))))))
1229 return std::pair(Ptr, AMDGPUAS::GLOBAL_ADDRESS);
1230
1231 return std::pair(nullptr, -1);
1232}
1233
1234unsigned
1235AMDGPUTargetMachine::getAddressSpaceForPseudoSourceKind(unsigned Kind) const {
1236 switch (Kind) {
1237 case PseudoSourceValue::Stack:
1238 case PseudoSourceValue::FixedStack:
1239 return AMDGPUAS::PRIVATE_ADDRESS;
1240 case PseudoSourceValue::ConstantPool:
1241 case PseudoSourceValue::GOT:
1242 case PseudoSourceValue::JumpTable:
1243 case PseudoSourceValue::GlobalValueCallEntry:
1244 case PseudoSourceValue::ExternalSymbolCallEntry:
1245 return AMDGPUAS::CONSTANT_ADDRESS;
1246 }
1247 return AMDGPUAS::FLAT_ADDRESS;
1248}
1249
1250bool AMDGPUTargetMachine::splitModule(
1251 Module &M, unsigned NumParts,
1252 function_ref<void(std::unique_ptr<Module> MPart)> ModuleCallback) {
1253 // FIXME(?): Would be better to use an already existing Analysis/PassManager,
1254 // but all current users of this API don't have one ready and would need to
1255 // create one anyway. Let's hide the boilerplate for now to keep it simple.
1256
1257 LoopAnalysisManager LAM;
1258 FunctionAnalysisManager FAM;
1259 CGSCCAnalysisManager CGAM;
1260 ModuleAnalysisManager MAM;
1261
1262 PassBuilder PB(this);
1263 PB.registerModuleAnalyses(MAM);
1264 PB.registerFunctionAnalyses(FAM);
1265 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
1266
1267 ModulePassManager MPM;
1268 MPM.addPass(Pass: AMDGPUSplitModulePass(NumParts, ModuleCallback));
1269 MPM.run(IR&: M, AM&: MAM);
1270 return true;
1271}
1272
1273//===----------------------------------------------------------------------===//
1274// GCN Target Machine (SI+)
1275//===----------------------------------------------------------------------===//
1276
1277GCNTargetMachine::GCNTargetMachine(const Target &T, const Triple &TT,
1278 StringRef CPU, StringRef FS,
1279 const TargetOptions &Options,
1280 std::optional<Reloc::Model> RM,
1281 std::optional<CodeModel::Model> CM,
1282 CodeGenOptLevel OL, bool JIT)
1283 : AMDGPUTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {
1284 setEnableDefaultMachineVerifier(false);
1285}
1286
1287enum class OOBFlagValue {
1288 Any = 0,
1289 Relaxed = 1,
1290 Strict = 2,
1291};
1292
1293/// Returns the OOB mode encoded by a module flag.
1294/// An absent flag defaults to Any.
1295static OOBFlagValue getOOBFlagValue(const Module &M, StringRef FlagName) {
1296 const auto *Flag =
1297 mdconst::dyn_extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: FlagName));
1298 if (!Flag)
1299 return OOBFlagValue::Any;
1300 return static_cast<OOBFlagValue>(Flag->getZExtValue());
1301}
1302
1303const TargetSubtargetInfo *
1304GCNTargetMachine::getSubtargetImpl(const Function &F) const {
1305 StringRef GPU = getGPUName(F);
1306 StringRef FS = getFeatureString(F);
1307
1308 const Module &M = *F.getParent();
1309 OOBFlagValue BufOOB = getOOBFlagValue(M, FlagName: AMDGPUOOBMode::BufferFlag);
1310 OOBFlagValue TBufOOB = getOOBFlagValue(M, FlagName: AMDGPUOOBMode::TBufferFlag);
1311 bool BufRelaxed = BufOOB == OOBFlagValue::Relaxed;
1312 bool TBufRelaxed = TBufOOB == OOBFlagValue::Relaxed;
1313 SmallString<128> SubtargetKey(GPU);
1314 SubtargetKey.append(RHS: FS);
1315 if (BufRelaxed)
1316 SubtargetKey.append(RHS: ",buf-oob=1");
1317 if (TBufRelaxed)
1318 SubtargetKey.append(RHS: ",tbuf-oob=1");
1319
1320 auto &I = SubtargetMap[SubtargetKey];
1321 if (!I) {
1322 AMDGPU::GPUKind Kind = AMDGPU::parseArchAMDGCN(CPU: GPU);
1323 Triple::SubArchType GPUSubArch = AMDGPU::getSubArch(AK: Kind);
1324
1325 // Enforce the subtarget is covered by the subarch. Tolerate no subarch for
1326 // legacy compatibility.
1327 const Triple &TT = M.getTargetTriple();
1328 if (GPUSubArch != TT.getSubArch() && Kind != AMDGPU::GK_NONE) {
1329 // Check if this is a generic subarch which has subtargets. Ignore
1330 // unknown subtargets with a known subarch, since for whatever reason
1331 // the convention is to just print a warning and ignore unrecognized
1332 // subtargets.
1333 bool IsLegacyEmptySubArch = TT.getSubArch() == Triple::NoSubArch;
1334 if (!IsLegacyEmptySubArch &&
1335 AMDGPU::getMajorSubArch(SubArch: GPUSubArch) != TT.getSubArch()) {
1336 F.getContext().emitError(ErrorStr: "invalid subtarget '" + Twine(GPU) +
1337 "' for subarch " + TT.getArchName());
1338 }
1339 }
1340
1341 I = std::make_unique<GCNSubtarget>(args: TargetTriple, args&: GPU, args&: FS, args: *this, args&: BufRelaxed,
1342 args&: TBufRelaxed);
1343 }
1344
1345 I->setScalarizeGlobalBehavior(ScalarizeGlobal);
1346
1347 return I.get();
1348}
1349
1350TargetTransformInfo
1351GCNTargetMachine::getTargetTransformInfo(const Function &F) const {
1352 return TargetTransformInfo(std::make_unique<GCNTTIImpl>(args: this, args: F));
1353}
1354
1355Error GCNTargetMachine::buildCodeGenPipeline(
1356 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
1357 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
1358 const CGPassBuilderOption &Opts, MCContext &Ctx,
1359 PassInstrumentationCallbacks *PIC) {
1360 AMDGPUCodeGenPassBuilder CGPB(*this, Opts, PIC);
1361 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
1362}
1363
1364ScheduleDAGInstrs *
1365GCNTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
1366 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
1367 if (ST.enableSIScheduler())
1368 return createSIMachineScheduler(C);
1369
1370 StringRef SchedStrategy = AMDGPU::getSchedStrategy(F: C->MF->getFunction());
1371
1372 if (SchedStrategy == "max-ilp")
1373 return createGCNMaxILPMachineScheduler(C);
1374
1375 if (SchedStrategy == "max-memory-clause")
1376 return createGCNMaxMemoryClauseMachineScheduler(C);
1377
1378 if (SchedStrategy == "iterative-ilp")
1379 return createIterativeILPMachineScheduler(C);
1380
1381 if (SchedStrategy == "iterative-minreg")
1382 return createMinRegScheduler(C);
1383
1384 if (SchedStrategy == "iterative-maxocc")
1385 return createIterativeGCNMaxOccupancyMachineScheduler(C);
1386
1387 if (SchedStrategy == "coexec") {
1388 diagnoseUnsupportedCoExecSchedulerSelection(F: C->MF->getFunction(), ST);
1389 return createGCNCoExecMachineScheduler(C);
1390 }
1391
1392 return createGCNMaxOccupancyMachineScheduler(C);
1393}
1394
1395ScheduleDAGInstrs *
1396GCNTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
1397 if (useNoopPostScheduler(F: C->MF->getFunction()))
1398 return createGCNNoopPostMachineScheduler(C);
1399
1400 ScheduleDAGMI *DAG =
1401 new GCNPostScheduleDAGMILive(C, std::make_unique<PostGenericScheduler>(args&: C),
1402 /*RemoveKillFlags=*/true);
1403 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
1404 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
1405 if (ST.shouldClusterStores())
1406 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
1407 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::PostRA));
1408 if ((EnableVOPD.getNumOccurrences() ||
1409 getOptLevel() >= CodeGenOptLevel::Less) &&
1410 EnableVOPD)
1411 DAG->addMutation(Mutation: createVOPDPairingMutation());
1412 DAG->addMutation(Mutation: createAMDGPUExportClusteringDAGMutation());
1413 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
1414 DAG->addMutation(Mutation: createAMDGPUHazardLatencyDAGMutation(MF: C->MF));
1415 return DAG;
1416}
1417//===----------------------------------------------------------------------===//
1418// AMDGPU Legacy Pass Setup
1419//===----------------------------------------------------------------------===//
1420
1421std::unique_ptr<CSEConfigBase> llvm::AMDGPUPassConfig::getCSEConfig() const {
1422 return getStandardCSEConfigForOpt(Level: TM->getOptLevel());
1423}
1424
1425namespace {
1426
1427class GCNPassConfig final : public AMDGPUPassConfig {
1428public:
1429 GCNPassConfig(TargetMachine &TM, PassManagerBase &PM)
1430 : AMDGPUPassConfig(TM, PM) {
1431 substitutePass(StandardID: &PostRASchedulerID, TargetID: &PostMachineSchedulerID);
1432 }
1433
1434 GCNTargetMachine &getGCNTargetMachine() const {
1435 return getTM<GCNTargetMachine>();
1436 }
1437
1438 bool addPreISel() override;
1439 void addMachineSSAOptimization() override;
1440 bool addILPOpts() override;
1441 bool addInstSelector() override;
1442 bool addIRTranslator() override;
1443 void addPreLegalizeMachineIR() override;
1444 bool addLegalizeMachineIR() override;
1445 void addPreRegBankSelect() override;
1446 bool addRegBankSelect() override;
1447 void addPreGlobalInstructionSelect() override;
1448 bool addGlobalInstructionSelect() override;
1449 void addPreRegAlloc() override;
1450 void addFastRegAlloc() override;
1451 void addOptimizedRegAlloc() override;
1452
1453 FunctionPass *createSGPRAllocPass(bool Optimized);
1454 FunctionPass *createVGPRAllocPass(bool Optimized);
1455 FunctionPass *createWWMRegAllocPass(bool Optimized);
1456 FunctionPass *createRegAllocPass(bool Optimized) override;
1457
1458 bool addRegAssignAndRewriteFast() override;
1459 bool addRegAssignAndRewriteOptimized() override;
1460
1461 bool addPreRewrite() override;
1462 void addPostRegAlloc() override;
1463 void addPreSched2() override;
1464 void addPreEmitPass() override;
1465 void addPostBBSections() override;
1466};
1467
1468} // end anonymous namespace
1469
1470AMDGPUPassConfig::AMDGPUPassConfig(TargetMachine &TM, PassManagerBase &PM)
1471 : TargetPassConfig(TM, PM) {
1472 // Exceptions and StackMaps are not supported, so these passes will never do
1473 // anything.
1474 disablePass(PassID: &StackMapLivenessID);
1475 disablePass(PassID: &FuncletLayoutID);
1476 // Garbage collection is not supported.
1477 disablePass(PassID: &GCLoweringID);
1478 disablePass(PassID: &ShadowStackGCLoweringID);
1479}
1480
1481void AMDGPUPassConfig::addEarlyCSEOrGVNPass() {
1482 if (getOptLevel() == CodeGenOptLevel::Aggressive)
1483 addPass(P: createGVNPass());
1484 else
1485 addPass(P: createEarlyCSEPass());
1486}
1487
1488void AMDGPUPassConfig::addStraightLineScalarOptimizationPasses() {
1489 if (isPassEnabled(Opt: EnableLoopPrefetch, Level: CodeGenOptLevel::Aggressive))
1490 addPass(P: createLoopDataPrefetchPass());
1491 addPass(P: createSeparateConstOffsetFromGEPPass());
1492 // ReassociateGEPs exposes more opportunities for SLSR. See
1493 // the example in reassociate-geps-and-slsr.ll.
1494 addPass(P: createStraightLineStrengthReducePass());
1495 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or
1496 // EarlyCSE can reuse.
1497 addEarlyCSEOrGVNPass();
1498 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
1499 addPass(P: createNaryReassociatePass());
1500 // NaryReassociate on GEPs creates redundant common expressions, so run
1501 // EarlyCSE after it.
1502 addPass(P: createEarlyCSEPass());
1503}
1504
1505void AMDGPUPassConfig::addIRPasses() {
1506 const AMDGPUTargetMachine &TM = getAMDGPUTargetMachine();
1507
1508 if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN())
1509 addPass(P: createAMDGPURemoveIncompatibleFunctionsPass(&TM));
1510
1511 // There is no reason to run these.
1512 disablePass(PassID: &StackMapLivenessID);
1513 disablePass(PassID: &FuncletLayoutID);
1514 disablePass(PassID: &PatchableFunctionID);
1515
1516 if (TM.getTargetTriple().isAMDGCN())
1517 addPass(P: createAMDGPUPrintfRuntimeBinding());
1518
1519 if (LowerCtorDtor)
1520 addPass(P: createAMDGPUCtorDtorLoweringLegacyPass());
1521
1522 if (TM.getTargetTriple().isAMDGCN() &&
1523 isPassEnabled(Opt: EnableImageIntrinsicOptimizer))
1524 addPass(P: createAMDGPUImageIntrinsicOptimizerPass(&TM));
1525
1526 if (EnableUniformIntrinsicCombine)
1527 addPass(P: createAMDGPUUniformIntrinsicCombineLegacyPass());
1528
1529 // This can be disabled by passing ::Disable here or on the command line
1530 // with --expand-variadics-override=disable.
1531 addPass(P: createExpandVariadicsPass(ExpandVariadicsMode::Lowering));
1532
1533 // Function calls are not supported, so make sure we inline everything.
1534 addPass(P: createAMDGPUAlwaysInlinePass());
1535 addPass(P: createAlwaysInlinerLegacyPass());
1536
1537 // Handle uses of OpenCL image2d_t, image3d_t and sampler_t arguments.
1538 if (TM.getTargetTriple().getArch() == Triple::r600)
1539 addPass(P: createR600OpenCLImageTypeLoweringPass());
1540
1541 // Make enqueued block runtime handles externally visible.
1542 addPass(P: createAMDGPUExportKernelRuntimeHandlesLegacyPass());
1543
1544 // Lower special LDS accesses.
1545 if (EnableLowerExecSync)
1546 addPass(P: createAMDGPULowerExecSyncLegacyPass());
1547
1548 // Lower LDS accesses to global memory pass if address sanitizer is enabled.
1549 if (EnableSwLowerLDS)
1550 addPass(P: createAMDGPUSwLowerLDSLegacyPass());
1551
1552 // Runs before PromoteAlloca so the latter can account for function uses
1553 if (EnableLowerModuleLDS) {
1554 addPass(P: createAMDGPULowerModuleLDSLegacyPass(TM: &TM));
1555 }
1556
1557 // Run atomic optimizer before Atomic Expand
1558 if ((TM.getTargetTriple().isAMDGCN()) &&
1559 (TM.getOptLevel() >= CodeGenOptLevel::Less) &&
1560 (AMDGPUAtomicOptimizerStrategy != ScanOptions::None)) {
1561 addPass(P: createAMDGPUAtomicOptimizerPass(ScanStrategy: AMDGPUAtomicOptimizerStrategy));
1562 }
1563
1564 addPass(P: createAtomicExpandLegacyPass());
1565
1566 if (TM.getOptLevel() > CodeGenOptLevel::None) {
1567 addPass(P: createAMDGPUPromoteAlloca());
1568
1569 if (isPassEnabled(Opt: EnableScalarIRPasses))
1570 addStraightLineScalarOptimizationPasses();
1571
1572 if (EnableAMDGPUAliasAnalysis) {
1573 addPass(P: createAMDGPUAAWrapperPass());
1574 addPass(P: createExternalAAWrapperPass(Callback: [](Pass &P, Function &,
1575 AAResults &AAR) {
1576 if (auto *WrapperPass = P.getAnalysisIfAvailable<AMDGPUAAWrapperPass>())
1577 AAR.addAAResult(AAResult&: WrapperPass->getResult());
1578 }));
1579 }
1580
1581 if (TM.getTargetTriple().isAMDGCN()) {
1582 // TODO: May want to move later or split into an early and late one.
1583 addPass(P: createAMDGPUCodeGenPreparePass());
1584 }
1585
1586 // Try to hoist loop invariant parts of divisions AMDGPUCodeGenPrepare may
1587 // have expanded.
1588 if (TM.getOptLevel() > CodeGenOptLevel::Less)
1589 addPass(P: createLICMPass());
1590 }
1591
1592 TargetPassConfig::addIRPasses();
1593
1594 // EarlyCSE is not always strong enough to clean up what LSR produces. For
1595 // example, GVN can combine
1596 //
1597 // %0 = add %a, %b
1598 // %1 = add %b, %a
1599 //
1600 // and
1601 //
1602 // %0 = shl nsw %a, 2
1603 // %1 = shl %a, 2
1604 //
1605 // but EarlyCSE can do neither of them.
1606 if (isPassEnabled(Opt: EnableScalarIRPasses))
1607 addEarlyCSEOrGVNPass();
1608}
1609
1610void AMDGPUPassConfig::addCodeGenPrepare() {
1611 if (TM->getTargetTriple().isAMDGCN() &&
1612 TM->getOptLevel() > CodeGenOptLevel::None)
1613 addPass(P: createAMDGPUPreloadKernelArgumentsLegacyPass(TM));
1614
1615 if (TM->getTargetTriple().isAMDGCN() && EnableLowerKernelArguments)
1616 addPass(P: createAMDGPULowerKernelArgumentsPass());
1617
1618 TargetPassConfig::addCodeGenPrepare();
1619
1620 if (isPassEnabled(Opt: EnableLoadStoreVectorizer))
1621 addPass(P: createLoadStoreVectorizerPass());
1622
1623 if (TM->getTargetTriple().isAMDGCN()) {
1624 // This lowering has been placed after codegenprepare to take advantage of
1625 // address mode matching (which is why it isn't put with the LDS lowerings).
1626 // It could be placed anywhere before uniformity annotations (an analysis
1627 // that it changes by splitting up fat pointers into their components)
1628 // but has been put before switch lowering and CFG flattening so that those
1629 // passes can run on the more optimized control flow this pass creates in
1630 // many cases.
1631 addPass(P: createAMDGPULowerBufferFatPointersPass());
1632 addPass(P: createAMDGPULowerIntrinsicsLegacyPass());
1633 }
1634
1635 // LowerSwitch pass may introduce unreachable blocks that can
1636 // cause unexpected behavior for subsequent passes. Placing it
1637 // here seems better that these blocks would get cleaned up by
1638 // UnreachableBlockElim inserted next in the pass flow.
1639 addPass(P: createLowerSwitchPass());
1640}
1641
1642bool AMDGPUPassConfig::addPreISel() {
1643 if (TM->getOptLevel() > CodeGenOptLevel::None)
1644 addPass(P: createFlattenCFGPass());
1645 return false;
1646}
1647
1648bool AMDGPUPassConfig::addInstSelector() {
1649 addPass(P: createAMDGPUISelDag(TM&: getAMDGPUTargetMachine(), OptLevel: getOptLevel()));
1650 return false;
1651}
1652
1653bool AMDGPUPassConfig::addGCPasses() {
1654 // Do nothing. GC is not supported.
1655 return false;
1656}
1657
1658//===----------------------------------------------------------------------===//
1659// GCN Legacy Pass Setup
1660//===----------------------------------------------------------------------===//
1661
1662bool GCNPassConfig::addPreISel() {
1663 AMDGPUPassConfig::addPreISel();
1664
1665 if (TM->getOptLevel() > CodeGenOptLevel::None) {
1666 addPass(P: createSinkingPass());
1667 addPass(P: createAMDGPULateCodeGenPrepareLegacyPass());
1668 }
1669
1670 // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
1671 // regions formed by them.
1672 addPass(PassID: &AMDGPUUnifyDivergentExitNodesID);
1673 addPass(P: createFixIrreduciblePass());
1674 addPass(P: createUnifyLoopExitsPass());
1675 addPass(P: createStructurizeCFGPass(SkipUniformRegions: false)); // true -> SkipUniformRegions
1676
1677 addPass(P: createAMDGPUAnnotateUniformValuesLegacy());
1678 addPass(P: createSIAnnotateControlFlowLegacyPass());
1679 // TODO: Move this right after structurizeCFG to avoid extra divergence
1680 // analysis. This depends on stopping SIAnnotateControlFlow from making
1681 // control flow modifications.
1682 addPass(P: createAMDGPURewriteUndefForPHILegacyPass());
1683
1684 // SDAG requires LCSSA, GlobalISel does not. Disable LCSSA for -global-isel
1685 // without any of the fallback options.
1686 if (getCGPassBuilderOption().EnableGlobalISelOption !=
1687 cl::boolOrDefault::BOU_TRUE ||
1688 !isGlobalISelAbortEnabled())
1689 addPass(P: createLCSSAPass());
1690
1691 if (TM->getOptLevel() > CodeGenOptLevel::Less)
1692 addPass(PassID: &AMDGPUPerfHintAnalysisLegacyID);
1693
1694 return false;
1695}
1696
1697void GCNPassConfig::addMachineSSAOptimization() {
1698 TargetPassConfig::addMachineSSAOptimization();
1699
1700 // We want to fold operands after PeepholeOptimizer has run (or as part of
1701 // it), because it will eliminate extra copies making it easier to fold the
1702 // real source operand. We want to eliminate dead instructions after, so that
1703 // we see fewer uses of the copies. We then need to clean up the dead
1704 // instructions leftover after the operands are folded as well.
1705 //
1706 // XXX - Can we get away without running DeadMachineInstructionElim again?
1707 addPass(PassID: &SIFoldOperandsLegacyID);
1708 if (EnableDPPCombine)
1709 addPass(PassID: &GCNDPPCombineLegacyID);
1710 addPass(PassID: &SILoadStoreOptimizerLegacyID);
1711 if (isPassEnabled(Opt: EnableSDWAPeephole)) {
1712 addPass(PassID: &SIPeepholeSDWALegacyID);
1713 addPass(PassID: &EarlyMachineLICMID);
1714 addPass(PassID: &MachineCSELegacyID);
1715 addPass(PassID: &SIFoldOperandsLegacyID);
1716 }
1717 addPass(PassID: &DeadMachineInstructionElimID);
1718 addPass(P: createSIShrinkInstructionsLegacyPass());
1719}
1720
1721bool GCNPassConfig::addILPOpts() {
1722 if (EnableEarlyIfConversion)
1723 addPass(PassID: &EarlyIfConverterLegacyID);
1724
1725 TargetPassConfig::addILPOpts();
1726 return false;
1727}
1728
1729bool GCNPassConfig::addInstSelector() {
1730 AMDGPUPassConfig::addInstSelector();
1731 addPass(PassID: &SIFixSGPRCopiesLegacyID);
1732 addPass(P: createSILowerI1CopiesLegacyPass());
1733 return false;
1734}
1735
1736bool GCNPassConfig::addIRTranslator() {
1737 addPass(P: new IRTranslator(getOptLevel()));
1738 return false;
1739}
1740
1741void GCNPassConfig::addPreLegalizeMachineIR() {
1742 bool IsOptNone = getOptLevel() == CodeGenOptLevel::None;
1743 addPass(P: createAMDGPUPreLegalizeCombiner(IsOptNone));
1744 addPass(P: new Localizer());
1745}
1746
1747bool GCNPassConfig::addLegalizeMachineIR() {
1748 addPass(P: new Legalizer());
1749 return false;
1750}
1751
1752void GCNPassConfig::addPreRegBankSelect() {
1753 bool IsOptNone = getOptLevel() == CodeGenOptLevel::None;
1754 addPass(P: createAMDGPUPostLegalizeCombiner(IsOptNone));
1755 addPass(P: createAMDGPUGlobalISelDivergenceLoweringPass());
1756}
1757
1758bool GCNPassConfig::addRegBankSelect() {
1759 addPass(P: createAMDGPURegBankSelectPass());
1760 addPass(P: createAMDGPURegBankLegalizePass());
1761 return false;
1762}
1763
1764void GCNPassConfig::addPreGlobalInstructionSelect() {
1765 bool IsOptNone = getOptLevel() == CodeGenOptLevel::None;
1766 addPass(P: createAMDGPURegBankCombiner(IsOptNone));
1767}
1768
1769bool GCNPassConfig::addGlobalInstructionSelect() {
1770 addPass(P: new InstructionSelect(getOptLevel()));
1771 return false;
1772}
1773
1774void GCNPassConfig::addFastRegAlloc() {
1775 // FIXME: We have to disable the verifier here because of PHIElimination +
1776 // TwoAddressInstructions disabling it.
1777
1778 // This must be run immediately after phi elimination and before
1779 // TwoAddressInstructions, otherwise the processing of the tied operand of
1780 // SI_ELSE will introduce a copy of the tied operand source after the else.
1781 insertPass(TargetPassID: &PHIEliminationID, InsertedPassID: &SILowerControlFlowLegacyID);
1782
1783 insertPass(TargetPassID: &TwoAddressInstructionPassID, InsertedPassID: &SIWholeQuadModeID);
1784
1785 TargetPassConfig::addFastRegAlloc();
1786}
1787
1788void GCNPassConfig::addPreRegAlloc() {
1789 if (getOptLevel() != CodeGenOptLevel::None)
1790 addPass(PassID: &AMDGPUPrepareAGPRAllocLegacyID);
1791}
1792
1793void GCNPassConfig::addOptimizedRegAlloc() {
1794 if (EnableDCEInRA)
1795 insertPass(TargetPassID: &DetectDeadLanesID, InsertedPassID: &DeadMachineInstructionElimID);
1796
1797 // FIXME: when an instruction has a Killed operand, and the instruction is
1798 // inside a bundle, seems only the BUNDLE instruction appears as the Kills of
1799 // the register in LiveVariables, this would trigger a failure in verifier,
1800 // we should fix it and enable the verifier.
1801 if (OptVGPRLiveRange)
1802 insertPass(TargetPassID: &LiveVariablesID, InsertedPassID: &SIOptimizeVGPRLiveRangeLegacyID);
1803
1804 // This must be run immediately after phi elimination and before
1805 // TwoAddressInstructions, otherwise the processing of the tied operand of
1806 // SI_ELSE will introduce a copy of the tied operand source after the else.
1807 insertPass(TargetPassID: &PHIEliminationID, InsertedPassID: &SILowerControlFlowLegacyID);
1808
1809 if (EnableRewritePartialRegUses)
1810 insertPass(TargetPassID: &RenameIndependentSubregsID, InsertedPassID: &GCNRewritePartialRegUsesID);
1811
1812 if (isPassEnabled(Opt: EnablePreRAOptimizations))
1813 insertPass(TargetPassID: &MachineSchedulerID, InsertedPassID: &GCNPreRAOptimizationsID);
1814
1815 // Allow the scheduler to run before SIWholeQuadMode inserts exec manipulation
1816 // instructions that cause scheduling barriers.
1817 insertPass(TargetPassID: &MachineSchedulerID, InsertedPassID: &SIWholeQuadModeID);
1818
1819 if (OptExecMaskPreRA)
1820 insertPass(TargetPassID: &MachineSchedulerID, InsertedPassID: &SIOptimizeExecMaskingPreRAID);
1821
1822 // This is not an essential optimization and it has a noticeable impact on
1823 // compilation time, so we only enable it from O2.
1824 if (TM->getOptLevel() > CodeGenOptLevel::Less)
1825 insertPass(TargetPassID: &MachineSchedulerID, InsertedPassID: &SIFormMemoryClausesID);
1826
1827 TargetPassConfig::addOptimizedRegAlloc();
1828}
1829
1830bool GCNPassConfig::addPreRewrite() {
1831 if (EnableRegReassign)
1832 addPass(PassID: &GCNNSAReassignID);
1833
1834 addPass(PassID: &AMDGPURewriteAGPRCopyMFMALegacyID);
1835 return true;
1836}
1837
1838FunctionPass *GCNPassConfig::createSGPRAllocPass(bool Optimized) {
1839 // Initialize the global default.
1840 llvm::call_once(flag&: InitializeDefaultSGPRRegisterAllocatorFlag,
1841 F&: initializeDefaultSGPRRegisterAllocatorOnce);
1842
1843 RegisterRegAlloc::FunctionPassCtor Ctor = SGPRRegisterRegAlloc::getDefault();
1844 if (Ctor != useDefaultRegisterAllocator)
1845 return Ctor();
1846
1847 if (Optimized)
1848 return createGreedyRegisterAllocator(F: onlyAllocateSGPRs);
1849
1850 return createFastRegisterAllocator(F: onlyAllocateSGPRs, ClearVirtRegs: false);
1851}
1852
1853FunctionPass *GCNPassConfig::createVGPRAllocPass(bool Optimized) {
1854 // Initialize the global default.
1855 llvm::call_once(flag&: InitializeDefaultVGPRRegisterAllocatorFlag,
1856 F&: initializeDefaultVGPRRegisterAllocatorOnce);
1857
1858 RegisterRegAlloc::FunctionPassCtor Ctor = VGPRRegisterRegAlloc::getDefault();
1859 if (Ctor != useDefaultRegisterAllocator)
1860 return Ctor();
1861
1862 if (Optimized)
1863 return createGreedyVGPRRegisterAllocator();
1864
1865 return createFastVGPRRegisterAllocator();
1866}
1867
1868FunctionPass *GCNPassConfig::createWWMRegAllocPass(bool Optimized) {
1869 // Initialize the global default.
1870 llvm::call_once(flag&: InitializeDefaultWWMRegisterAllocatorFlag,
1871 F&: initializeDefaultWWMRegisterAllocatorOnce);
1872
1873 RegisterRegAlloc::FunctionPassCtor Ctor = WWMRegisterRegAlloc::getDefault();
1874 if (Ctor != useDefaultRegisterAllocator)
1875 return Ctor();
1876
1877 if (Optimized)
1878 return createGreedyWWMRegisterAllocator();
1879
1880 return createFastWWMRegisterAllocator();
1881}
1882
1883FunctionPass *GCNPassConfig::createRegAllocPass(bool Optimized) {
1884 llvm_unreachable("should not be used");
1885}
1886
1887static const char RegAllocOptNotSupportedMessage[] =
1888 "-regalloc not supported with amdgcn. Use -sgpr-regalloc, -wwm-regalloc, "
1889 "and -vgpr-regalloc";
1890
1891bool GCNPassConfig::addRegAssignAndRewriteFast() {
1892 if (!usingDefaultRegAlloc())
1893 reportFatalUsageError(reason: RegAllocOptNotSupportedMessage);
1894
1895 addPass(PassID: &GCNPreRALongBranchRegID);
1896
1897 addPass(P: createSGPRAllocPass(Optimized: false));
1898
1899 // Equivalent of PEI for SGPRs.
1900 addPass(PassID: &SILowerSGPRSpillsLegacyID);
1901
1902 // To Allocate wwm registers used in whole quad mode operations (for shaders).
1903 addPass(PassID: &SIPreAllocateWWMRegsLegacyID);
1904
1905 // For allocating other wwm register operands.
1906 addPass(P: createWWMRegAllocPass(Optimized: false));
1907
1908 addPass(PassID: &SILowerWWMCopiesLegacyID);
1909 addPass(PassID: &AMDGPUReserveWWMRegsLegacyID);
1910
1911 // For allocating per-thread VGPRs.
1912 addPass(P: createVGPRAllocPass(Optimized: false));
1913
1914 return true;
1915}
1916
1917bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
1918 if (!usingDefaultRegAlloc())
1919 reportFatalUsageError(reason: RegAllocOptNotSupportedMessage);
1920
1921 addPass(PassID: &GCNPreRALongBranchRegID);
1922
1923 addPass(P: createSGPRAllocPass(Optimized: true));
1924
1925 // Commit allocated register changes. This is mostly necessary because too
1926 // many things rely on the use lists of the physical registers, such as the
1927 // verifier. This is only necessary with allocators which use LiveIntervals,
1928 // since FastRegAlloc does the replacements itself.
1929 addPass(P: createVirtRegRewriter(ClearVirtRegs: false));
1930
1931 // At this point, the sgpr-regalloc has been done and it is good to have the
1932 // stack slot coloring to try to optimize the SGPR spill stack indices before
1933 // attempting the custom SGPR spill lowering.
1934 addPass(PassID: &StackSlotColoringID);
1935
1936 // Equivalent of PEI for SGPRs.
1937 addPass(PassID: &SILowerSGPRSpillsLegacyID);
1938
1939 // To Allocate wwm registers used in whole quad mode operations (for shaders).
1940 addPass(PassID: &SIPreAllocateWWMRegsLegacyID);
1941
1942 // For allocating other whole wave mode registers.
1943 addPass(P: createWWMRegAllocPass(Optimized: true));
1944 addPass(PassID: &SILowerWWMCopiesLegacyID);
1945 addPass(P: createVirtRegRewriter(ClearVirtRegs: false));
1946 addPass(PassID: &AMDGPUReserveWWMRegsLegacyID);
1947
1948 // For allocating per-thread VGPRs.
1949 addPass(P: createVGPRAllocPass(Optimized: true));
1950
1951 addPreRewrite();
1952 addPass(PassID: &VirtRegRewriterID);
1953
1954 addPass(PassID: &AMDGPUMarkLastScratchLoadID);
1955
1956 return true;
1957}
1958
1959void GCNPassConfig::addPostRegAlloc() {
1960 addPass(PassID: &SIFixVGPRCopiesID);
1961 if (getOptLevel() > CodeGenOptLevel::None)
1962 addPass(PassID: &SIOptimizeExecMaskingLegacyID);
1963 TargetPassConfig::addPostRegAlloc();
1964}
1965
1966void GCNPassConfig::addPreSched2() {
1967 if (TM->getOptLevel() > CodeGenOptLevel::None)
1968 addPass(P: createSIShrinkInstructionsLegacyPass());
1969 addPass(PassID: &SIPostRABundlerLegacyID);
1970}
1971
1972void GCNPassConfig::addPreEmitPass() {
1973 if (isPassEnabled(Opt: EnableVOPD, Level: CodeGenOptLevel::Less))
1974 addPass(PassID: &GCNCreateVOPDID);
1975 addPass(P: createSIMemoryLegalizerPass());
1976 addPass(P: createSIInsertWaitcntsPass());
1977
1978 addPass(P: createSIModeRegisterPass());
1979
1980 if (getOptLevel() > CodeGenOptLevel::None)
1981 addPass(PassID: &SIInsertHardClausesID);
1982
1983 addPass(PassID: &SILateBranchLoweringPassID);
1984 if (isPassEnabled(Opt: EnableSetWavePriority, Level: CodeGenOptLevel::Less))
1985 addPass(P: createAMDGPUSetWavePriorityPass());
1986 if (getOptLevel() > CodeGenOptLevel::None)
1987 addPass(PassID: &SIPreEmitPeepholeID);
1988 // The hazard recognizer that runs as part of the post-ra scheduler does not
1989 // guarantee to be able handle all hazards correctly. This is because if there
1990 // are multiple scheduling regions in a basic block, the regions are scheduled
1991 // bottom up, so when we begin to schedule a region we don't know what
1992 // instructions were emitted directly before it.
1993 //
1994 // Here we add a stand-alone hazard recognizer pass which can handle all
1995 // cases.
1996 addPass(PassID: &PostRAHazardRecognizerID);
1997
1998 addPass(PassID: &AMDGPUWaitSGPRHazardsLegacyID);
1999
2000 addPass(PassID: &AMDGPULowerVGPREncodingLegacyID);
2001
2002 if (isPassEnabled(Opt: EnableInsertDelayAlu, Level: CodeGenOptLevel::Less))
2003 addPass(PassID: &AMDGPUInsertDelayAluID);
2004
2005 addPass(PassID: &BranchRelaxationPassID);
2006}
2007
2008void GCNPassConfig::addPostBBSections() {
2009 // We run this later to avoid passes like livedebugvalues and BBSections
2010 // having to deal with the apparent multi-entry functions we may generate.
2011 addPass(P: createAMDGPUPreloadKernArgPrologLegacyPass());
2012}
2013
2014TargetPassConfig *GCNTargetMachine::createPassConfig(PassManagerBase &PM) {
2015 return new GCNPassConfig(*this, PM);
2016}
2017
2018void GCNTargetMachine::registerMachineRegisterInfoCallback(
2019 MachineFunction &MF) const {
2020 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2021 MF.getRegInfo().addDelegate(delegate: MFI);
2022}
2023
2024MachineFunctionInfo *GCNTargetMachine::createMachineFunctionInfo(
2025 BumpPtrAllocator &Allocator, const Function &F,
2026 const TargetSubtargetInfo *STI) const {
2027 return SIMachineFunctionInfo::create<SIMachineFunctionInfo>(
2028 Allocator, F, STI: static_cast<const GCNSubtarget *>(STI));
2029}
2030
2031yaml::MachineFunctionInfo *GCNTargetMachine::createDefaultFuncInfoYAML() const {
2032 return new yaml::SIMachineFunctionInfo();
2033}
2034
2035yaml::MachineFunctionInfo *
2036GCNTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
2037 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2038 return new yaml::SIMachineFunctionInfo(
2039 *MFI, *MF.getSubtarget<GCNSubtarget>().getRegisterInfo(), MF);
2040}
2041
2042bool GCNTargetMachine::parseMachineFunctionInfo(
2043 const yaml::MachineFunctionInfo &MFI_, PerFunctionMIParsingState &PFS,
2044 SMDiagnostic &Error, SMRange &SourceRange) const {
2045 const yaml::SIMachineFunctionInfo &YamlMFI =
2046 static_cast<const yaml::SIMachineFunctionInfo &>(MFI_);
2047 MachineFunction &MF = PFS.MF;
2048 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2049 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2050
2051 if (MFI->initializeBaseYamlFields(YamlMFI, MF, PFS, Error, SourceRange))
2052 return true;
2053
2054 if (MFI->Occupancy == 0) {
2055 // Fixup the subtarget dependent default value.
2056 MFI->Occupancy = ST.getOccupancyWithWorkGroupSizes(MF).second;
2057 }
2058
2059 auto parseRegister = [&](const yaml::StringValue &RegName, Register &RegVal) {
2060 Register TempReg;
2061 if (parseNamedRegisterReference(PFS, Reg&: TempReg, Src: RegName.Value, Error)) {
2062 SourceRange = RegName.SourceRange;
2063 return true;
2064 }
2065 RegVal = TempReg;
2066
2067 return false;
2068 };
2069
2070 auto parseOptionalRegister = [&](const yaml::StringValue &RegName,
2071 Register &RegVal) {
2072 return !RegName.Value.empty() && parseRegister(RegName, RegVal);
2073 };
2074
2075 if (parseOptionalRegister(YamlMFI.VGPRForAGPRCopy, MFI->VGPRForAGPRCopy))
2076 return true;
2077
2078 if (parseOptionalRegister(YamlMFI.SGPRForEXECCopy, MFI->SGPRForEXECCopy))
2079 return true;
2080
2081 if (parseOptionalRegister(YamlMFI.LongBranchReservedReg,
2082 MFI->LongBranchReservedReg))
2083 return true;
2084
2085 auto diagnoseRegisterClass = [&](const yaml::StringValue &RegName) {
2086 // Create a diagnostic for a the register string literal.
2087 const MemoryBuffer &Buffer =
2088 *PFS.SM->getMemoryBuffer(i: PFS.SM->getMainFileID());
2089 Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1,
2090 RegName.Value.size(), SourceMgr::DK_Error,
2091 "incorrect register class for field", RegName.Value,
2092 {}, {});
2093 SourceRange = RegName.SourceRange;
2094 return true;
2095 };
2096
2097 if (parseRegister(YamlMFI.ScratchRSrcReg, MFI->ScratchRSrcReg) ||
2098 parseRegister(YamlMFI.FrameOffsetReg, MFI->FrameOffsetReg) ||
2099 parseRegister(YamlMFI.StackPtrOffsetReg, MFI->StackPtrOffsetReg))
2100 return true;
2101
2102 if (MFI->ScratchRSrcReg != AMDGPU::PRIVATE_RSRC_REG &&
2103 !AMDGPU::SGPR_128RegClass.contains(Reg: MFI->ScratchRSrcReg)) {
2104 return diagnoseRegisterClass(YamlMFI.ScratchRSrcReg);
2105 }
2106
2107 if (MFI->FrameOffsetReg != AMDGPU::FP_REG &&
2108 !AMDGPU::SGPR_32RegClass.contains(Reg: MFI->FrameOffsetReg)) {
2109 return diagnoseRegisterClass(YamlMFI.FrameOffsetReg);
2110 }
2111
2112 if (MFI->StackPtrOffsetReg != AMDGPU::SP_REG &&
2113 !AMDGPU::SGPR_32RegClass.contains(Reg: MFI->StackPtrOffsetReg)) {
2114 return diagnoseRegisterClass(YamlMFI.StackPtrOffsetReg);
2115 }
2116
2117 for (const auto &YamlReg : YamlMFI.WWMReservedRegs) {
2118 Register ParsedReg;
2119 if (parseRegister(YamlReg, ParsedReg))
2120 return true;
2121
2122 MFI->reserveWWMRegister(Reg: ParsedReg);
2123 }
2124
2125 for (const auto &[_, Info] : PFS.VRegInfosNamed) {
2126 MFI->setFlag(Reg: Info->VReg, Flag: Info->Flags);
2127 }
2128 for (const auto &[_, Info] : PFS.VRegInfos) {
2129 MFI->setFlag(Reg: Info->VReg, Flag: Info->Flags);
2130 }
2131
2132 for (const auto &YamlRegStr : YamlMFI.SpillPhysVGPRS) {
2133 Register ParsedReg;
2134 if (parseRegister(YamlRegStr, ParsedReg))
2135 return true;
2136 MFI->SpillPhysVGPRs.push_back(Elt: ParsedReg);
2137 }
2138
2139 auto parseAndCheckArgument = [&](const std::optional<yaml::SIArgument> &A,
2140 const TargetRegisterClass &RC,
2141 ArgDescriptor &Arg, unsigned UserSGPRs,
2142 unsigned SystemSGPRs) {
2143 // Skip parsing if it's not present.
2144 if (!A)
2145 return false;
2146
2147 if (A->IsRegister) {
2148 Register Reg;
2149 if (parseNamedRegisterReference(PFS, Reg, Src: A->RegisterName.Value, Error)) {
2150 SourceRange = A->RegisterName.SourceRange;
2151 return true;
2152 }
2153 if (!RC.contains(Reg))
2154 return diagnoseRegisterClass(A->RegisterName);
2155 Arg = ArgDescriptor::createRegister(Reg);
2156 } else
2157 Arg = ArgDescriptor::createStack(Offset: A->StackOffset);
2158 // Check and apply the optional mask.
2159 if (A->Mask)
2160 Arg = ArgDescriptor::createArg(Arg, Mask: *A->Mask);
2161
2162 MFI->NumUserSGPRs += UserSGPRs;
2163 MFI->NumSystemSGPRs += SystemSGPRs;
2164 return false;
2165 };
2166
2167 if (YamlMFI.ArgInfo &&
2168 (parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentBuffer,
2169 AMDGPU::SGPR_128RegClass,
2170 MFI->ArgInfo.PrivateSegmentBuffer, 4, 0) ||
2171 parseAndCheckArgument(YamlMFI.ArgInfo->DispatchPtr,
2172 AMDGPU::SReg_64RegClass, MFI->ArgInfo.DispatchPtr,
2173 2, 0) ||
2174 parseAndCheckArgument(YamlMFI.ArgInfo->QueuePtr, AMDGPU::SReg_64RegClass,
2175 MFI->ArgInfo.QueuePtr, 2, 0) ||
2176 parseAndCheckArgument(YamlMFI.ArgInfo->KernargSegmentPtr,
2177 AMDGPU::SReg_64RegClass,
2178 MFI->ArgInfo.KernargSegmentPtr, 2, 0) ||
2179 parseAndCheckArgument(YamlMFI.ArgInfo->DispatchID,
2180 AMDGPU::SReg_64RegClass, MFI->ArgInfo.DispatchID,
2181 2, 0) ||
2182 parseAndCheckArgument(YamlMFI.ArgInfo->FlatScratchInit,
2183 AMDGPU::SReg_64RegClass,
2184 MFI->ArgInfo.FlatScratchInit, 2, 0) ||
2185 parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentSize,
2186 AMDGPU::SGPR_32RegClass,
2187 MFI->ArgInfo.PrivateSegmentSize, 0, 0) ||
2188 parseAndCheckArgument(YamlMFI.ArgInfo->LDSKernelId,
2189 AMDGPU::SGPR_32RegClass,
2190 MFI->ArgInfo.LDSKernelId, 0, 1) ||
2191 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDX,
2192 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDX,
2193 0, 1) ||
2194 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDY,
2195 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDY,
2196 0, 1) ||
2197 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDZ,
2198 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDZ,
2199 0, 1) ||
2200 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupInfo,
2201 AMDGPU::SGPR_32RegClass,
2202 MFI->ArgInfo.WorkGroupInfo, 0, 1) ||
2203 parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentWaveByteOffset,
2204 AMDGPU::SGPR_32RegClass,
2205 MFI->ArgInfo.PrivateSegmentWaveByteOffset, 0, 1) ||
2206 parseAndCheckArgument(YamlMFI.ArgInfo->ImplicitArgPtr,
2207 AMDGPU::SReg_64RegClass,
2208 MFI->ArgInfo.ImplicitArgPtr, 0, 0) ||
2209 parseAndCheckArgument(YamlMFI.ArgInfo->ImplicitBufferPtr,
2210 AMDGPU::SReg_64RegClass,
2211 MFI->ArgInfo.ImplicitBufferPtr, 2, 0) ||
2212 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDX,
2213 AMDGPU::VGPR_32RegClass,
2214 MFI->ArgInfo.WorkItemIDX, 0, 0) ||
2215 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDY,
2216 AMDGPU::VGPR_32RegClass,
2217 MFI->ArgInfo.WorkItemIDY, 0, 0) ||
2218 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDZ,
2219 AMDGPU::VGPR_32RegClass,
2220 MFI->ArgInfo.WorkItemIDZ, 0, 0)))
2221 return true;
2222
2223 // Parse FirstKernArgPreloadReg separately, since it's a Register,
2224 // not ArgDescriptor.
2225 if (YamlMFI.ArgInfo && YamlMFI.ArgInfo->FirstKernArgPreloadReg) {
2226 const yaml::SIArgument &A = *YamlMFI.ArgInfo->FirstKernArgPreloadReg;
2227
2228 if (!A.IsRegister) {
2229 // For stack arguments, we don't have RegisterName.SourceRange,
2230 // but we should have some location info from the YAML parser
2231 const MemoryBuffer &Buffer =
2232 *PFS.SM->getMemoryBuffer(i: PFS.SM->getMainFileID());
2233 // Create a minimal valid source range
2234 SMLoc Loc = SMLoc::getFromPointer(Ptr: Buffer.getBufferStart());
2235 SMRange Range(Loc, Loc);
2236
2237 Error = SMDiagnostic(
2238 *PFS.SM, Loc, Buffer.getBufferIdentifier(), 1, 0, SourceMgr::DK_Error,
2239 "firstKernArgPreloadReg must be a register, not a stack location", "",
2240 {}, {});
2241
2242 SourceRange = Range;
2243 return true;
2244 }
2245
2246 Register Reg;
2247 if (parseNamedRegisterReference(PFS, Reg, Src: A.RegisterName.Value, Error)) {
2248 SourceRange = A.RegisterName.SourceRange;
2249 return true;
2250 }
2251
2252 if (!AMDGPU::SGPR_32RegClass.contains(Reg))
2253 return diagnoseRegisterClass(A.RegisterName);
2254
2255 MFI->ArgInfo.FirstKernArgPreloadReg = Reg;
2256 MFI->NumUserSGPRs += YamlMFI.NumKernargPreloadSGPRs;
2257 }
2258
2259 if (ST.hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode)) {
2260 MFI->Mode.IEEE = YamlMFI.Mode.IEEE;
2261 MFI->Mode.DX10Clamp = YamlMFI.Mode.DX10Clamp;
2262 }
2263
2264 // FIXME: Move proper support for denormal-fp-math into base MachineFunction
2265 MFI->Mode.FP32Denormals.Input = YamlMFI.Mode.FP32InputDenormals
2266 ? DenormalMode::IEEE
2267 : DenormalMode::PreserveSign;
2268 MFI->Mode.FP32Denormals.Output = YamlMFI.Mode.FP32OutputDenormals
2269 ? DenormalMode::IEEE
2270 : DenormalMode::PreserveSign;
2271
2272 MFI->Mode.FP64FP16Denormals.Input = YamlMFI.Mode.FP64FP16InputDenormals
2273 ? DenormalMode::IEEE
2274 : DenormalMode::PreserveSign;
2275 MFI->Mode.FP64FP16Denormals.Output = YamlMFI.Mode.FP64FP16OutputDenormals
2276 ? DenormalMode::IEEE
2277 : DenormalMode::PreserveSign;
2278
2279 if (YamlMFI.HasInitWholeWave)
2280 MFI->setInitWholeWave();
2281
2282 return false;
2283}
2284
2285//===----------------------------------------------------------------------===//
2286// AMDGPU CodeGen Pass Builder interface.
2287//===----------------------------------------------------------------------===//
2288
2289AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
2290 GCNTargetMachine &TM, const CGPassBuilderOption &Opts,
2291 PassInstrumentationCallbacks *PIC)
2292 : CodeGenPassBuilder(TM, Opts, PIC) {
2293 Opt.MISchedPostRA = true;
2294 Opt.RequiresCodeGenSCCOrder = true;
2295 // Exceptions and StackMaps are not supported, so these passes will never do
2296 // anything.
2297 // Garbage collection is not supported.
2298 disablePass<StackMapLivenessPass, FuncletLayoutPass, PatchableFunctionPass,
2299 ShadowStackGCLoweringPass, GCLoweringPass>();
2300}
2301
2302void AMDGPUCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) const {
2303 if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN()) {
2304 flushFPMsToMPM(PMW);
2305 addModulePass(Pass: AMDGPURemoveIncompatibleFunctionsPass(TM), PMW);
2306 }
2307
2308 flushFPMsToMPM(PMW);
2309
2310 if (TM.getTargetTriple().isAMDGCN())
2311 addModulePass(Pass: AMDGPUPrintfRuntimeBindingPass(), PMW);
2312
2313 if (LowerCtorDtor)
2314 addModulePass(Pass: AMDGPUCtorDtorLoweringPass(), PMW);
2315
2316 if (isPassEnabled(Opt: EnableImageIntrinsicOptimizer))
2317 addFunctionPass(Pass: AMDGPUImageIntrinsicOptimizerPass(TM), PMW);
2318
2319 if (EnableUniformIntrinsicCombine)
2320 addFunctionPass(Pass: AMDGPUUniformIntrinsicCombinePass(), PMW);
2321 // This can be disabled by passing ::Disable here or on the command line
2322 // with --expand-variadics-override=disable.
2323 flushFPMsToMPM(PMW);
2324 addModulePass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Lowering), PMW);
2325
2326 addModulePass(Pass: AMDGPUAlwaysInlinePass(), PMW);
2327 addModulePass(Pass: AlwaysInlinerPass(), PMW);
2328
2329 addModulePass(Pass: AMDGPUExportKernelRuntimeHandlesPass(), PMW);
2330
2331 if (EnableLowerExecSync)
2332 addModulePass(Pass: AMDGPULowerExecSyncPass(), PMW);
2333
2334 if (EnableSwLowerLDS)
2335 addModulePass(Pass: AMDGPUSwLowerLDSPass(), PMW);
2336
2337 // Runs before PromoteAlloca so the latter can account for function uses
2338 if (EnableLowerModuleLDS)
2339 addModulePass(Pass: AMDGPULowerModuleLDSPass(TM), PMW);
2340
2341 // Run atomic optimizer before Atomic Expand
2342 if (TM.getOptLevel() >= CodeGenOptLevel::Less &&
2343 (AMDGPUAtomicOptimizerStrategy != ScanOptions::None))
2344 addFunctionPass(
2345 Pass: AMDGPUAtomicOptimizerPass(TM, AMDGPUAtomicOptimizerStrategy), PMW);
2346
2347 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
2348
2349 if (TM.getOptLevel() > CodeGenOptLevel::None) {
2350 addFunctionPass(Pass: AMDGPUPromoteAllocaPass(TM), PMW);
2351 if (isPassEnabled(Opt: EnableScalarIRPasses))
2352 addStraightLineScalarOptimizationPasses(PMW);
2353
2354 // TODO: Handle EnableAMDGPUAliasAnalysis
2355
2356 // TODO: May want to move later or split into an early and late one.
2357 addFunctionPass(Pass: AMDGPUCodeGenPreparePass(TM), PMW);
2358
2359 // Try to hoist loop invariant parts of divisions AMDGPUCodeGenPrepare may
2360 // have expanded.
2361 if (TM.getOptLevel() > CodeGenOptLevel::Less) {
2362 addFunctionPass(Pass: createFunctionToLoopPassAdaptor(Pass: LICMPass(LICMOptions()),
2363 /*UseMemorySSA=*/true),
2364 PMW);
2365 }
2366 }
2367
2368 Base::addIRPasses(PMW);
2369
2370 // EarlyCSE is not always strong enough to clean up what LSR produces. For
2371 // example, GVN can combine
2372 //
2373 // %0 = add %a, %b
2374 // %1 = add %b, %a
2375 //
2376 // and
2377 //
2378 // %0 = shl nsw %a, 2
2379 // %1 = shl %a, 2
2380 //
2381 // but EarlyCSE can do neither of them.
2382 if (isPassEnabled(Opt: EnableScalarIRPasses))
2383 addEarlyCSEOrGVNPass(PMW);
2384}
2385
2386void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(
2387 PassManagerWrapper &PMW) const {
2388 if (TM.getOptLevel() > CodeGenOptLevel::None) {
2389 flushFPMsToMPM(PMW);
2390 addModulePass(Pass: AMDGPUPreloadKernelArgumentsPass(TM), PMW);
2391 }
2392
2393 if (EnableLowerKernelArguments)
2394 addFunctionPass(Pass: AMDGPULowerKernelArgumentsPass(TM), PMW);
2395
2396 Base::addCodeGenPrepare(PMW);
2397
2398 if (isPassEnabled(Opt: EnableLoadStoreVectorizer))
2399 addFunctionPass(Pass: LoadStoreVectorizerPass(), PMW);
2400
2401 // This lowering has been placed after codegenprepare to take advantage of
2402 // address mode matching (which is why it isn't put with the LDS lowerings).
2403 // It could be placed anywhere before uniformity annotations (an analysis
2404 // that it changes by splitting up fat pointers into their components)
2405 // but has been put before switch lowering and CFG flattening so that those
2406 // passes can run on the more optimized control flow this pass creates in
2407 // many cases.
2408 flushFPMsToMPM(PMW);
2409 addModulePass(Pass: AMDGPULowerBufferFatPointersPass(TM), PMW);
2410 flushFPMsToMPM(PMW);
2411 requireCGSCCOrder(PMW);
2412
2413 addModulePass(Pass: AMDGPULowerIntrinsicsPass(TM), PMW);
2414
2415 // LowerSwitch pass may introduce unreachable blocks that can cause unexpected
2416 // behavior for subsequent passes. Placing it here seems better that these
2417 // blocks would get cleaned up by UnreachableBlockElim inserted next in the
2418 // pass flow.
2419 addFunctionPass(Pass: LowerSwitchPass(), PMW);
2420}
2421
2422void AMDGPUCodeGenPassBuilder::addPreISel(PassManagerWrapper &PMW) const {
2423
2424 if (TM.getOptLevel() > CodeGenOptLevel::None) {
2425 addFunctionPass(Pass: FlattenCFGPass(), PMW);
2426 addFunctionPass(Pass: SinkingPass(), PMW);
2427 addFunctionPass(Pass: AMDGPULateCodeGenPreparePass(TM), PMW);
2428 }
2429
2430 // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
2431 // regions formed by them.
2432
2433 addFunctionPass(Pass: AMDGPUUnifyDivergentExitNodesPass(), PMW);
2434 addFunctionPass(Pass: FixIrreduciblePass(), PMW);
2435 addFunctionPass(Pass: UnifyLoopExitsPass(), PMW);
2436 addFunctionPass(Pass: StructurizeCFGPass(/*SkipUniformRegions=*/false), PMW);
2437
2438 addFunctionPass(Pass: AMDGPUAnnotateUniformValuesPass(), PMW);
2439
2440 addFunctionPass(Pass: SIAnnotateControlFlowPass(TM), PMW);
2441
2442 // TODO: Move this right after structurizeCFG to avoid extra divergence
2443 // analysis. This depends on stopping SIAnnotateControlFlow from making
2444 // control flow modifications.
2445 addFunctionPass(Pass: AMDGPURewriteUndefForPHIPass(), PMW);
2446
2447 if (getCGPassBuilderOption().EnableGlobalISelOption !=
2448 cl::boolOrDefault::BOU_TRUE ||
2449 !isGlobalISelAbortEnabled())
2450 addFunctionPass(Pass: LCSSAPass(), PMW);
2451
2452 if (TM.getOptLevel() > CodeGenOptLevel::Less) {
2453 flushFPMsToMPM(PMW);
2454 addModulePass(Pass: AMDGPUPerfHintAnalysisPass(TM), PMW);
2455 }
2456
2457 // FIXME: Why isn't this queried as required from AMDGPUISelDAGToDAG, and why
2458 // isn't this in addInstSelector?
2459 addFunctionPass(Pass: RequireAnalysisPass<UniformityInfoAnalysis, Function>(), PMW,
2460 /*Force=*/true);
2461}
2462
2463void AMDGPUCodeGenPassBuilder::addILPOpts(PassManagerWrapper &PMW) const {
2464 if (EnableEarlyIfConversion)
2465 addMachineFunctionPass(Pass: EarlyIfConverterPass(), PMW);
2466
2467 Base::addILPOpts(PMW);
2468}
2469
2470void AMDGPUCodeGenPassBuilder::addAsmPrinterBegin(
2471 PassManagerWrapper &PMW) const {
2472 addModulePass(Pass: AMDGPUAsmPrinterBeginPass(), PMW,
2473 /*Force=*/true);
2474}
2475
2476void AMDGPUCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) const {
2477 addMachineFunctionPass(Pass: AMDGPUAsmPrinterPass(), PMW);
2478}
2479
2480void AMDGPUCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) const {
2481 addModulePass(Pass: AMDGPUAsmPrinterEndPass(), PMW);
2482}
2483
2484Error AMDGPUCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) const {
2485 addMachineFunctionPass(Pass: AMDGPUISelDAGToDAGPass(TM), PMW);
2486 addMachineFunctionPass(Pass: SIFixSGPRCopiesPass(), PMW);
2487 addMachineFunctionPass(Pass: SILowerI1CopiesPass(), PMW);
2488 return Error::success();
2489}
2490
2491void AMDGPUCodeGenPassBuilder::addPreRewrite(PassManagerWrapper &PMW) const {
2492 if (EnableRegReassign) {
2493 addMachineFunctionPass(Pass: GCNNSAReassignPass(), PMW);
2494 }
2495
2496 addMachineFunctionPass(Pass: AMDGPURewriteAGPRCopyMFMAPass(), PMW);
2497}
2498
2499void AMDGPUCodeGenPassBuilder::addMachineSSAOptimization(
2500 PassManagerWrapper &PMW) const {
2501 Base::addMachineSSAOptimization(PMW);
2502
2503 addMachineFunctionPass(Pass: SIFoldOperandsPass(), PMW);
2504 if (EnableDPPCombine) {
2505 addMachineFunctionPass(Pass: GCNDPPCombinePass(), PMW);
2506 }
2507 addMachineFunctionPass(Pass: SILoadStoreOptimizerPass(), PMW);
2508 if (isPassEnabled(Opt: EnableSDWAPeephole)) {
2509 addMachineFunctionPass(Pass: SIPeepholeSDWAPass(), PMW);
2510 addMachineFunctionPass(Pass: EarlyMachineLICMPass(), PMW);
2511 addMachineFunctionPass(Pass: MachineCSEPass(), PMW);
2512 addMachineFunctionPass(Pass: SIFoldOperandsPass(), PMW);
2513 }
2514 addMachineFunctionPass(Pass: DeadMachineInstructionElimPass(), PMW);
2515 addMachineFunctionPass(Pass: SIShrinkInstructionsPass(), PMW);
2516}
2517
2518Error AMDGPUCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) const {
2519 insertPass<PHIEliminationPass>(Pass: SILowerControlFlowPass());
2520
2521 insertPass<TwoAddressInstructionPass>(Pass: SIWholeQuadModePass());
2522
2523 return Base::addFastRegAlloc(PMW);
2524}
2525
2526Error AMDGPUCodeGenPassBuilder::addRegAssignmentFast(
2527 PassManagerWrapper &PMW) const {
2528 if (auto Err = validateRegAllocOptions())
2529 return Err;
2530
2531 addMachineFunctionPass(Pass: GCNPreRALongBranchRegPass(), PMW);
2532
2533 // SGPR allocation - default to fast at -O0.
2534 if (SGPRRegAllocNPM == RegAllocType::Greedy)
2535 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateSGPRs, "sgpr"}), PMW);
2536 else
2537 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateSGPRs, "sgpr", false}),
2538 PMW);
2539
2540 // Equivalent of PEI for SGPRs.
2541 addMachineFunctionPass(Pass: SILowerSGPRSpillsPass(), PMW);
2542
2543 // To Allocate wwm registers used in whole quad mode operations (for shaders).
2544 addMachineFunctionPass(Pass: SIPreAllocateWWMRegsPass(), PMW);
2545
2546 // WWM allocation - default to fast at -O0.
2547 if (WWMRegAllocNPM == RegAllocType::Greedy)
2548 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateWWMRegs, "wwm"}), PMW);
2549 else
2550 addMachineFunctionPass(
2551 Pass: RegAllocFastPass({onlyAllocateWWMRegs, "wwm", false}), PMW);
2552
2553 addMachineFunctionPass(Pass: SILowerWWMCopiesPass(), PMW);
2554 addMachineFunctionPass(Pass: AMDGPUReserveWWMRegsPass(), PMW);
2555
2556 // VGPR allocation - default to fast at -O0.
2557 if (VGPRRegAllocNPM == RegAllocType::Greedy)
2558 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2559 else
2560 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2561
2562 return Error::success();
2563}
2564
2565Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(
2566 PassManagerWrapper &PMW) const {
2567 if (EnableDCEInRA)
2568 insertPass<DetectDeadLanesPass>(Pass: DeadMachineInstructionElimPass());
2569
2570 // FIXME: when an instruction has a Killed operand, and the instruction is
2571 // inside a bundle, seems only the BUNDLE instruction appears as the Kills of
2572 // the register in LiveVariables, this would trigger a failure in verifier,
2573 // we should fix it and enable the verifier.
2574 if (OptVGPRLiveRange)
2575 insertPass<RequireAnalysisPass<LiveVariablesAnalysis, MachineFunction>>(
2576 Pass: SIOptimizeVGPRLiveRangePass());
2577
2578 // This must be run immediately after phi elimination and before
2579 // TwoAddressInstructions, otherwise the processing of the tied operand of
2580 // SI_ELSE will introduce a copy of the tied operand source after the else.
2581 insertPass<PHIEliminationPass>(Pass: SILowerControlFlowPass());
2582
2583 if (EnableRewritePartialRegUses)
2584 insertPass<RenameIndependentSubregsPass>(Pass: GCNRewritePartialRegUsesPass());
2585
2586 if (isPassEnabled(Opt: EnablePreRAOptimizations))
2587 insertPass<MachineSchedulerPass>(Pass: GCNPreRAOptimizationsPass());
2588
2589 // Allow the scheduler to run before SIWholeQuadMode inserts exec manipulation
2590 // instructions that cause scheduling barriers.
2591 insertPass<MachineSchedulerPass>(Pass: SIWholeQuadModePass());
2592
2593 if (OptExecMaskPreRA)
2594 insertPass<MachineSchedulerPass>(Pass: SIOptimizeExecMaskingPreRAPass());
2595
2596 // This is not an essential optimization and it has a noticeable impact on
2597 // compilation time, so we only enable it from O2.
2598 if (TM.getOptLevel() > CodeGenOptLevel::Less)
2599 insertPass<MachineSchedulerPass>(Pass: SIFormMemoryClausesPass());
2600
2601 return Base::addOptimizedRegAlloc(PMW);
2602}
2603
2604void AMDGPUCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) const {
2605 if (getOptLevel() != CodeGenOptLevel::None)
2606 addMachineFunctionPass(Pass: AMDGPUPrepareAGPRAllocPass(), PMW);
2607}
2608
2609Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized(
2610 PassManagerWrapper &PMW) const {
2611 if (auto Err = validateRegAllocOptions())
2612 return Err;
2613
2614 addMachineFunctionPass(Pass: GCNPreRALongBranchRegPass(), PMW);
2615
2616 // SGPR allocation - default to greedy at -O1 and above.
2617 if (SGPRRegAllocNPM == RegAllocType::Fast)
2618 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateSGPRs, "sgpr", false}),
2619 PMW);
2620 else
2621 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateSGPRs, "sgpr"}), PMW);
2622
2623 // Commit allocated register changes. This is mostly necessary because too
2624 // many things rely on the use lists of the physical registers, such as the
2625 // verifier. This is only necessary with allocators which use LiveIntervals,
2626 // since FastRegAlloc does the replacements itself.
2627 addMachineFunctionPass(Pass: VirtRegRewriterPass(false), PMW);
2628
2629 // At this point, the sgpr-regalloc has been done and it is good to have the
2630 // stack slot coloring to try to optimize the SGPR spill stack indices before
2631 // attempting the custom SGPR spill lowering.
2632 addMachineFunctionPass(Pass: StackSlotColoringPass(), PMW);
2633
2634 // Equivalent of PEI for SGPRs.
2635 addMachineFunctionPass(Pass: SILowerSGPRSpillsPass(), PMW);
2636
2637 // To Allocate wwm registers used in whole quad mode operations (for shaders).
2638 addMachineFunctionPass(Pass: SIPreAllocateWWMRegsPass(), PMW);
2639
2640 // WWM allocation - default to greedy at -O1 and above.
2641 if (WWMRegAllocNPM == RegAllocType::Fast)
2642 addMachineFunctionPass(
2643 Pass: RegAllocFastPass({onlyAllocateWWMRegs, "wwm", false}), PMW);
2644 else
2645 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateWWMRegs, "wwm"}), PMW);
2646 addMachineFunctionPass(Pass: SILowerWWMCopiesPass(), PMW);
2647 addMachineFunctionPass(Pass: VirtRegRewriterPass(false), PMW);
2648 addMachineFunctionPass(Pass: AMDGPUReserveWWMRegsPass(), PMW);
2649
2650 // VGPR allocation - default to greedy at -O1 and above.
2651 if (VGPRRegAllocNPM == RegAllocType::Fast)
2652 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2653 else
2654 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2655
2656 addPreRewrite(PMW);
2657 addMachineFunctionPass(Pass: VirtRegRewriterPass(true), PMW);
2658
2659 addMachineFunctionPass(Pass: AMDGPUMarkLastScratchLoadPass(), PMW);
2660 return Error::success();
2661}
2662
2663void AMDGPUCodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) const {
2664 addMachineFunctionPass(Pass: SIFixVGPRCopiesPass(), PMW);
2665 if (TM.getOptLevel() > CodeGenOptLevel::None)
2666 addMachineFunctionPass(Pass: SIOptimizeExecMaskingPass(), PMW);
2667 Base::addPostRegAlloc(PMW);
2668}
2669
2670void AMDGPUCodeGenPassBuilder::addPreSched2(PassManagerWrapper &PMW) const {
2671 if (TM.getOptLevel() > CodeGenOptLevel::None)
2672 addMachineFunctionPass(Pass: SIShrinkInstructionsPass(), PMW);
2673 addMachineFunctionPass(Pass: SIPostRABundlerPass(), PMW);
2674}
2675
2676void AMDGPUCodeGenPassBuilder::addPostBBSections(
2677 PassManagerWrapper &PMW) const {
2678 // We run this later to avoid passes like livedebugvalues and BBSections
2679 // having to deal with the apparent multi-entry functions we may generate.
2680 addMachineFunctionPass(Pass: AMDGPUPreloadKernArgPrologPass(), PMW);
2681}
2682
2683void AMDGPUCodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) const {
2684 if (isPassEnabled(Opt: EnableVOPD, Level: CodeGenOptLevel::Less)) {
2685 addMachineFunctionPass(Pass: GCNCreateVOPDPass(), PMW);
2686 }
2687
2688 addMachineFunctionPass(Pass: SIMemoryLegalizerPass(), PMW);
2689 addMachineFunctionPass(Pass: SIInsertWaitcntsPass(), PMW);
2690
2691 addMachineFunctionPass(Pass: SIModeRegisterPass(), PMW);
2692
2693 if (TM.getOptLevel() > CodeGenOptLevel::None)
2694 addMachineFunctionPass(Pass: SIInsertHardClausesPass(), PMW);
2695
2696 addMachineFunctionPass(Pass: SILateBranchLoweringPass(), PMW);
2697
2698 if (isPassEnabled(Opt: EnableSetWavePriority, Level: CodeGenOptLevel::Less))
2699 addMachineFunctionPass(Pass: AMDGPUSetWavePriorityPass(), PMW);
2700
2701 if (TM.getOptLevel() > CodeGenOptLevel::None)
2702 addMachineFunctionPass(Pass: SIPreEmitPeepholePass(), PMW);
2703
2704 // The hazard recognizer that runs as part of the post-ra scheduler does not
2705 // guarantee to be able handle all hazards correctly. This is because if there
2706 // are multiple scheduling regions in a basic block, the regions are scheduled
2707 // bottom up, so when we begin to schedule a region we don't know what
2708 // instructions were emitted directly before it.
2709 //
2710 // Here we add a stand-alone hazard recognizer pass which can handle all
2711 // cases.
2712 addMachineFunctionPass(Pass: PostRAHazardRecognizerPass(), PMW);
2713 addMachineFunctionPass(Pass: AMDGPUWaitSGPRHazardsPass(), PMW);
2714 addMachineFunctionPass(Pass: AMDGPULowerVGPREncodingPass(), PMW);
2715
2716 if (isPassEnabled(Opt: EnableInsertDelayAlu, Level: CodeGenOptLevel::Less)) {
2717 addMachineFunctionPass(Pass: AMDGPUInsertDelayAluPass(), PMW);
2718 }
2719
2720 addMachineFunctionPass(Pass: BranchRelaxationPass(), PMW);
2721}
2722
2723bool AMDGPUCodeGenPassBuilder::isPassEnabled(const cl::opt<bool> &Opt,
2724 CodeGenOptLevel Level) const {
2725 if (Opt.getNumOccurrences())
2726 return Opt;
2727 if (TM.getOptLevel() < Level)
2728 return false;
2729 return Opt;
2730}
2731
2732void AMDGPUCodeGenPassBuilder::addEarlyCSEOrGVNPass(
2733 PassManagerWrapper &PMW) const {
2734 if (TM.getOptLevel() == CodeGenOptLevel::Aggressive)
2735 addFunctionPass(Pass: GVNPass(), PMW);
2736 else
2737 addFunctionPass(Pass: EarlyCSEPass(), PMW);
2738}
2739
2740void AMDGPUCodeGenPassBuilder::addStraightLineScalarOptimizationPasses(
2741 PassManagerWrapper &PMW) const {
2742 if (isPassEnabled(Opt: EnableLoopPrefetch, Level: CodeGenOptLevel::Aggressive))
2743 addFunctionPass(Pass: LoopDataPrefetchPass(), PMW);
2744
2745 addFunctionPass(Pass: SeparateConstOffsetFromGEPPass(), PMW);
2746
2747 // ReassociateGEPs exposes more opportunities for SLSR. See
2748 // the example in reassociate-geps-and-slsr.ll.
2749 addFunctionPass(Pass: StraightLineStrengthReducePass(), PMW);
2750
2751 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or
2752 // EarlyCSE can reuse.
2753 addEarlyCSEOrGVNPass(PMW);
2754
2755 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
2756 addFunctionPass(Pass: NaryReassociatePass(), PMW);
2757
2758 // NaryReassociate on GEPs creates redundant common expressions, so run
2759 // EarlyCSE after it.
2760 addFunctionPass(Pass: EarlyCSEPass(), PMW);
2761}
2762