1//===-- AMDGPUTargetMachine.cpp - TargetMachine for hw codegen targets-----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file contains both AMDGPU target machine and the CodeGen pass builder.
11/// The AMDGPU target machine contains all of the hardware specific information
12/// needed to emit code for SI+ GPUs in the legacy pass manager pipeline. The
13/// CodeGen pass builder handles the pass pipeline for new pass manager.
14//
15//===----------------------------------------------------------------------===//
16
17#include "AMDGPUTargetMachine.h"
18#include "AMDGPU.h"
19#include "AMDGPUAliasAnalysis.h"
20#include "AMDGPUAsmPrinter.h"
21#include "AMDGPUBarrierLatency.h"
22#include "AMDGPUCoExecSchedStrategy.h"
23#include "AMDGPUCtorDtorLowering.h"
24#include "AMDGPUExportClustering.h"
25#include "AMDGPUExportKernelRuntimeHandles.h"
26#include "AMDGPUHazardLatency.h"
27#include "AMDGPUIGroupLP.h"
28#include "AMDGPUISelDAGToDAG.h"
29#include "AMDGPULowerVGPREncoding.h"
30#include "AMDGPUMacroFusion.h"
31#include "AMDGPUNextUseAnalysis.h"
32#include "AMDGPUPerfHintAnalysis.h"
33#include "AMDGPUPreloadKernArgProlog.h"
34#include "AMDGPUPrepareAGPRAlloc.h"
35#include "AMDGPURemoveIncompatibleFunctions.h"
36#include "AMDGPUReserveWWMRegs.h"
37#include "AMDGPUResourceUsageAnalysis.h"
38#include "AMDGPUSplitModule.h"
39#include "AMDGPUTargetObjectFile.h"
40#include "AMDGPUTargetTransformInfo.h"
41#include "AMDGPUUnifyDivergentExitNodes.h"
42#include "AMDGPUWaitSGPRHazards.h"
43#include "GCNDPPCombine.h"
44#include "GCNIterativeScheduler.h"
45#include "GCNNSAReassign.h"
46#include "GCNPreRALongBranchReg.h"
47#include "GCNPreRAOptimizations.h"
48#include "GCNRewritePartialRegUses.h"
49#include "GCNSchedStrategy.h"
50#include "GCNVOPDUtils.h"
51#include "R600.h"
52#include "R600TargetMachine.h"
53#include "SIFixSGPRCopies.h"
54#include "SIFixVGPRCopies.h"
55#include "SIFoldOperands.h"
56#include "SIFormMemoryClauses.h"
57#include "SILoadStoreOptimizer.h"
58#include "SILowerControlFlow.h"
59#include "SILowerSGPRSpills.h"
60#include "SILowerWWMCopies.h"
61#include "SIMachineFunctionInfo.h"
62#include "SIMachineScheduler.h"
63#include "SIOptimizeExecMasking.h"
64#include "SIOptimizeExecMaskingPreRA.h"
65#include "SIOptimizeVGPRLiveRange.h"
66#include "SIPeepholeSDWA.h"
67#include "SIPostRABundler.h"
68#include "SIPreAllocateWWMRegs.h"
69#include "SIShrinkInstructions.h"
70#include "SIWholeQuadMode.h"
71#include "TargetInfo/AMDGPUTargetInfo.h"
72#include "Utils/AMDGPUBaseInfo.h"
73#include "llvm/Analysis/CGSCCPassManager.h"
74#include "llvm/Analysis/CallGraphSCCPass.h"
75#include "llvm/Analysis/KernelInfo.h"
76#include "llvm/Analysis/UniformityAnalysis.h"
77#include "llvm/CodeGen/AtomicExpand.h"
78#include "llvm/CodeGen/BranchRelaxation.h"
79#include "llvm/CodeGen/DeadMachineInstructionElim.h"
80#include "llvm/CodeGen/DetectDeadLanes.h"
81#include "llvm/CodeGen/EarlyIfConversion.h"
82#include "llvm/CodeGen/FuncletLayout.h"
83#include "llvm/CodeGen/GCMetadata.h"
84#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
85#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
86#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
87#include "llvm/CodeGen/GlobalISel/Legalizer.h"
88#include "llvm/CodeGen/GlobalISel/Localizer.h"
89#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
90#include "llvm/CodeGen/MIRParser/MIParser.h"
91#include "llvm/CodeGen/MachineCSE.h"
92#include "llvm/CodeGen/MachineLICM.h"
93#include "llvm/CodeGen/MachineScheduler.h"
94#include "llvm/CodeGen/PHIElimination.h"
95#include "llvm/CodeGen/Passes.h"
96#include "llvm/CodeGen/PatchableFunction.h"
97#include "llvm/CodeGen/PostRAHazardRecognizer.h"
98#include "llvm/CodeGen/RegAllocFast.h"
99#include "llvm/CodeGen/RegAllocGreedyPass.h"
100#include "llvm/CodeGen/RegAllocRegistry.h"
101#include "llvm/CodeGen/RenameIndependentSubregs.h"
102#include "llvm/CodeGen/ShadowStackGCLowering.h"
103#include "llvm/CodeGen/StackSlotColoring.h"
104#include "llvm/CodeGen/TargetPassConfig.h"
105#include "llvm/CodeGen/TwoAddressInstructionPass.h"
106#include "llvm/IR/DiagnosticInfo.h"
107#include "llvm/IR/IntrinsicsAMDGPU.h"
108#include "llvm/IR/Module.h"
109#include "llvm/IR/PassManager.h"
110#include "llvm/IR/PatternMatch.h"
111#include "llvm/InitializePasses.h"
112#include "llvm/MC/TargetRegistry.h"
113#include "llvm/Passes/CodeGenPassBuilder.h"
114#include "llvm/Passes/PassBuilder.h"
115#include "llvm/Support/Compiler.h"
116#include "llvm/Support/FormatVariadic.h"
117#include "llvm/TargetParser/AMDGPUTargetParser.h"
118#include "llvm/Transforms/HipStdPar/HipStdPar.h"
119#include "llvm/Transforms/IPO.h"
120#include "llvm/Transforms/IPO/AlwaysInliner.h"
121#include "llvm/Transforms/IPO/ExpandVariadics.h"
122#include "llvm/Transforms/IPO/GlobalDCE.h"
123#include "llvm/Transforms/IPO/Internalize.h"
124#include "llvm/Transforms/Scalar.h"
125#include "llvm/Transforms/Scalar/EarlyCSE.h"
126#include "llvm/Transforms/Scalar/FlattenCFG.h"
127#include "llvm/Transforms/Scalar/GVN.h"
128#include "llvm/Transforms/Scalar/InferAddressSpaces.h"
129#include "llvm/Transforms/Scalar/LICM.h"
130#include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
131#include "llvm/Transforms/Scalar/LoopPassManager.h"
132#include "llvm/Transforms/Scalar/NaryReassociate.h"
133#include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"
134#include "llvm/Transforms/Scalar/Sink.h"
135#include "llvm/Transforms/Scalar/StraightLineStrengthReduce.h"
136#include "llvm/Transforms/Scalar/StructurizeCFG.h"
137#include "llvm/Transforms/Utils.h"
138#include "llvm/Transforms/Utils/FixIrreducible.h"
139#include "llvm/Transforms/Utils/LCSSA.h"
140#include "llvm/Transforms/Utils/LowerSwitch.h"
141#include "llvm/Transforms/Utils/SimplifyLibCalls.h"
142#include "llvm/Transforms/Utils/UnifyLoopExits.h"
143#include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
144#include <optional>
145
146using namespace llvm;
147using namespace llvm::PatternMatch;
148
149namespace {
150//===----------------------------------------------------------------------===//
151// AMDGPU CodeGen Pass Builder interface.
152//===----------------------------------------------------------------------===//
153
154class AMDGPUCodeGenPassBuilder : public CodeGenPassBuilder {
155 using Base = CodeGenPassBuilder;
156
157 GCNTargetMachine &getTM() const {
158 return static_cast<GCNTargetMachine &>(TM);
159 }
160
161public:
162 AMDGPUCodeGenPassBuilder(GCNTargetMachine &TM,
163 const CGPassBuilderOption &Opts,
164 PassInstrumentationCallbacks *PIC);
165
166 void addIRPasses(PassManagerWrapper &PMW) override;
167 void addCodeGenPrepare(PassManagerWrapper &PMW) override;
168 void addPreISel(PassManagerWrapper &PMW) override;
169 void addILPOpts(PassManagerWrapper &PMW) override;
170 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
171 void addAsmPrinter(PassManagerWrapper &PMW) override;
172 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
173 Error addInstSelector(PassManagerWrapper &PMW) override;
174 void addPreRewrite(PassManagerWrapper &PMW) override;
175 void addMachineSSAOptimization(PassManagerWrapper &PMW) override;
176 void addPostRegAlloc(PassManagerWrapper &PMW) override;
177 void addPreEmitPass(PassManagerWrapper &PMW) override;
178 Error addRegAssignAndRewriteFast(PassManagerWrapper &PMW) override;
179 Expected<bool>
180 addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) override;
181 void addPreRegAlloc(PassManagerWrapper &PMW) override;
182 Error addFastRegAlloc(PassManagerWrapper &PMW) override;
183 Error addOptimizedRegAlloc(PassManagerWrapper &PMW) override;
184 void addPreSched2(PassManagerWrapper &PMW) override;
185 void addPostBBSections(PassManagerWrapper &PMW) override;
186
187private:
188 Error validateRegAllocOptions() const;
189
190public:
191 /// Check if a pass is enabled given \p Opt option. The option always
192 /// overrides defaults if explicitly used. Otherwise its default will be used
193 /// given that a pass shall work at an optimization \p Level minimum.
194 bool isPassEnabled(const cl::opt<bool> &Opt,
195 CodeGenOptLevel Level = CodeGenOptLevel::Default) const;
196 void addEarlyCSEOrGVNPass(PassManagerWrapper &PMW);
197 void addStraightLineScalarOptimizationPasses(PassManagerWrapper &PMW);
198};
199
200class SGPRRegisterRegAlloc : public RegisterRegAllocBase<SGPRRegisterRegAlloc> {
201public:
202 SGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
203 : RegisterRegAllocBase(N, D, C) {}
204};
205
206class VGPRRegisterRegAlloc : public RegisterRegAllocBase<VGPRRegisterRegAlloc> {
207public:
208 VGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
209 : RegisterRegAllocBase(N, D, C) {}
210};
211
212class WWMRegisterRegAlloc : public RegisterRegAllocBase<WWMRegisterRegAlloc> {
213public:
214 WWMRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
215 : RegisterRegAllocBase(N, D, C) {}
216};
217
218static bool onlyAllocateSGPRs(const TargetRegisterInfo &TRI,
219 const MachineRegisterInfo &MRI,
220 const Register Reg) {
221 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
222 return static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC);
223}
224
225static bool onlyAllocateVGPRs(const TargetRegisterInfo &TRI,
226 const MachineRegisterInfo &MRI,
227 const Register Reg) {
228 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
229 return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC);
230}
231
232static bool onlyAllocateWWMRegs(const TargetRegisterInfo &TRI,
233 const MachineRegisterInfo &MRI,
234 const Register Reg) {
235 const SIMachineFunctionInfo *MFI =
236 MRI.getMF().getInfo<SIMachineFunctionInfo>();
237 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
238 return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC) &&
239 MFI->checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG);
240}
241
242/// -{sgpr|wwm|vgpr}-regalloc=... command line option.
243static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
244
245/// A dummy default pass factory indicates whether the register allocator is
246/// overridden on the command line.
247static llvm::once_flag InitializeDefaultSGPRRegisterAllocatorFlag;
248static llvm::once_flag InitializeDefaultVGPRRegisterAllocatorFlag;
249static llvm::once_flag InitializeDefaultWWMRegisterAllocatorFlag;
250
251static SGPRRegisterRegAlloc
252defaultSGPRRegAlloc("default",
253 "pick SGPR register allocator based on -O option",
254 useDefaultRegisterAllocator);
255
256static cl::opt<SGPRRegisterRegAlloc::FunctionPassCtor, false,
257 RegisterPassParser<SGPRRegisterRegAlloc>>
258SGPRRegAlloc("sgpr-regalloc", cl::Hidden, cl::init(Val: &useDefaultRegisterAllocator),
259 cl::desc("Register allocator to use for SGPRs"));
260
261static cl::opt<VGPRRegisterRegAlloc::FunctionPassCtor, false,
262 RegisterPassParser<VGPRRegisterRegAlloc>>
263VGPRRegAlloc("vgpr-regalloc", cl::Hidden, cl::init(Val: &useDefaultRegisterAllocator),
264 cl::desc("Register allocator to use for VGPRs"));
265
266static cl::opt<WWMRegisterRegAlloc::FunctionPassCtor, false,
267 RegisterPassParser<WWMRegisterRegAlloc>>
268 WWMRegAlloc("wwm-regalloc", cl::Hidden,
269 cl::init(Val: &useDefaultRegisterAllocator),
270 cl::desc("Register allocator to use for WWM registers"));
271
272// New pass manager register allocator options for AMDGPU
273static cl::opt<RegAllocType, false, RegAllocTypeParser> SGPRRegAllocNPM(
274 "sgpr-regalloc-npm", cl::Hidden, cl::init(Val: RegAllocType::Default),
275 cl::desc("Register allocator for SGPRs (new pass manager)"));
276
277static cl::opt<RegAllocType, false, RegAllocTypeParser> VGPRRegAllocNPM(
278 "vgpr-regalloc-npm", cl::Hidden, cl::init(Val: RegAllocType::Default),
279 cl::desc("Register allocator for VGPRs (new pass manager)"));
280
281static cl::opt<RegAllocType, false, RegAllocTypeParser> WWMRegAllocNPM(
282 "wwm-regalloc-npm", cl::Hidden, cl::init(Val: RegAllocType::Default),
283 cl::desc("Register allocator for WWM registers (new pass manager)"));
284
285/// Check if the given RegAllocType is supported for AMDGPU NPM register
286/// allocation. Only Fast and Greedy are supported; Basic and PBQP are not.
287static Error checkRegAllocSupported(RegAllocType RAType, StringRef RegName) {
288 if (RAType == RegAllocType::Basic || RAType == RegAllocType::PBQP) {
289 return make_error<StringError>(
290 Args: Twine("unsupported register allocator '") +
291 (RAType == RegAllocType::Basic ? "basic" : "pbqp") + "' for " +
292 RegName + " registers",
293 Args: inconvertibleErrorCode());
294 }
295 return Error::success();
296}
297
298Error AMDGPUCodeGenPassBuilder::validateRegAllocOptions() const {
299 // 1. Generic --regalloc-npm is not supported for AMDGPU.
300 if (Opt.RegAlloc != RegAllocType::Unset) {
301 return make_error<StringError>(
302 Args: "-regalloc-npm not supported for amdgcn. Use -sgpr-regalloc-npm, "
303 "-vgpr-regalloc-npm, and -wwm-regalloc-npm",
304 Args: inconvertibleErrorCode());
305 }
306
307 // 2. Legacy PM regalloc options are not compatible with NPM.
308 if (SGPRRegAlloc.getNumOccurrences() > 0 ||
309 VGPRRegAlloc.getNumOccurrences() > 0 ||
310 WWMRegAlloc.getNumOccurrences() > 0) {
311 return make_error<StringError>(
312 Args: "-sgpr-regalloc, -vgpr-regalloc, and -wwm-regalloc are legacy PM "
313 "options. Use -sgpr-regalloc-npm, -vgpr-regalloc-npm, and "
314 "-wwm-regalloc-npm with the new pass manager",
315 Args: inconvertibleErrorCode());
316 }
317
318 // 3. Only Fast and Greedy allocators are supported for AMDGPU.
319 if (auto Err = checkRegAllocSupported(RAType: SGPRRegAllocNPM, RegName: "SGPR"))
320 return Err;
321 if (auto Err = checkRegAllocSupported(RAType: WWMRegAllocNPM, RegName: "WWM"))
322 return Err;
323 if (auto Err = checkRegAllocSupported(RAType: VGPRRegAllocNPM, RegName: "VGPR"))
324 return Err;
325
326 return Error::success();
327}
328
329static void initializeDefaultSGPRRegisterAllocatorOnce() {
330 RegisterRegAlloc::FunctionPassCtor Ctor = SGPRRegisterRegAlloc::getDefault();
331
332 if (!Ctor) {
333 Ctor = SGPRRegAlloc;
334 SGPRRegisterRegAlloc::setDefault(SGPRRegAlloc);
335 }
336}
337
338static void initializeDefaultVGPRRegisterAllocatorOnce() {
339 RegisterRegAlloc::FunctionPassCtor Ctor = VGPRRegisterRegAlloc::getDefault();
340
341 if (!Ctor) {
342 Ctor = VGPRRegAlloc;
343 VGPRRegisterRegAlloc::setDefault(VGPRRegAlloc);
344 }
345}
346
347static void initializeDefaultWWMRegisterAllocatorOnce() {
348 RegisterRegAlloc::FunctionPassCtor Ctor = WWMRegisterRegAlloc::getDefault();
349
350 if (!Ctor) {
351 Ctor = WWMRegAlloc;
352 WWMRegisterRegAlloc::setDefault(WWMRegAlloc);
353 }
354}
355
356static FunctionPass *createBasicSGPRRegisterAllocator() {
357 return createBasicRegisterAllocator(F: onlyAllocateSGPRs);
358}
359
360static FunctionPass *createGreedySGPRRegisterAllocator() {
361 return createGreedyRegisterAllocator(F: onlyAllocateSGPRs);
362}
363
364static FunctionPass *createFastSGPRRegisterAllocator() {
365 return createFastRegisterAllocator(F: onlyAllocateSGPRs, ClearVirtRegs: false);
366}
367
368static FunctionPass *createBasicVGPRRegisterAllocator() {
369 return createBasicRegisterAllocator(F: onlyAllocateVGPRs);
370}
371
372static FunctionPass *createGreedyVGPRRegisterAllocator() {
373 return createGreedyRegisterAllocator(F: onlyAllocateVGPRs);
374}
375
376static FunctionPass *createFastVGPRRegisterAllocator() {
377 return createFastRegisterAllocator(F: onlyAllocateVGPRs, ClearVirtRegs: true);
378}
379
380static FunctionPass *createBasicWWMRegisterAllocator() {
381 return createBasicRegisterAllocator(F: onlyAllocateWWMRegs);
382}
383
384static FunctionPass *createGreedyWWMRegisterAllocator() {
385 return createGreedyRegisterAllocator(F: onlyAllocateWWMRegs);
386}
387
388static FunctionPass *createFastWWMRegisterAllocator() {
389 return createFastRegisterAllocator(F: onlyAllocateWWMRegs, ClearVirtRegs: false);
390}
391
392static SGPRRegisterRegAlloc basicRegAllocSGPR(
393 "basic", "basic register allocator", createBasicSGPRRegisterAllocator);
394static SGPRRegisterRegAlloc greedyRegAllocSGPR(
395 "greedy", "greedy register allocator", createGreedySGPRRegisterAllocator);
396
397static SGPRRegisterRegAlloc fastRegAllocSGPR(
398 "fast", "fast register allocator", createFastSGPRRegisterAllocator);
399
400
401static VGPRRegisterRegAlloc basicRegAllocVGPR(
402 "basic", "basic register allocator", createBasicVGPRRegisterAllocator);
403static VGPRRegisterRegAlloc greedyRegAllocVGPR(
404 "greedy", "greedy register allocator", createGreedyVGPRRegisterAllocator);
405
406static VGPRRegisterRegAlloc fastRegAllocVGPR(
407 "fast", "fast register allocator", createFastVGPRRegisterAllocator);
408static WWMRegisterRegAlloc basicRegAllocWWMReg("basic",
409 "basic register allocator",
410 createBasicWWMRegisterAllocator);
411static WWMRegisterRegAlloc
412 greedyRegAllocWWMReg("greedy", "greedy register allocator",
413 createGreedyWWMRegisterAllocator);
414static WWMRegisterRegAlloc fastRegAllocWWMReg("fast", "fast register allocator",
415 createFastWWMRegisterAllocator);
416
417static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {
418 return Phase == ThinOrFullLTOPhase::FullLTOPreLink ||
419 Phase == ThinOrFullLTOPhase::ThinLTOPreLink;
420}
421} // anonymous namespace
422
423static cl::opt<bool>
424EnableEarlyIfConversion("amdgpu-early-ifcvt", cl::Hidden,
425 cl::desc("Run early if-conversion"),
426 cl::init(Val: false));
427
428static cl::opt<bool>
429OptExecMaskPreRA("amdgpu-opt-exec-mask-pre-ra", cl::Hidden,
430 cl::desc("Run pre-RA exec mask optimizations"),
431 cl::init(Val: true));
432
433static cl::opt<bool>
434 LowerCtorDtor("amdgpu-lower-global-ctor-dtor",
435 cl::desc("Lower GPU ctor / dtors to globals on the device."),
436 cl::init(Val: true), cl::Hidden);
437
438// Option to disable vectorizer for tests.
439static cl::opt<bool> EnableLoadStoreVectorizer(
440 "amdgpu-load-store-vectorizer",
441 cl::desc("Enable load store vectorizer"),
442 cl::init(Val: true),
443 cl::Hidden);
444
445// Option to control global loads scalarization
446static cl::opt<bool> ScalarizeGlobal(
447 "amdgpu-scalarize-global-loads",
448 cl::desc("Enable global load scalarization"),
449 cl::init(Val: true),
450 cl::Hidden);
451
452// Option to run internalize pass.
453static cl::opt<bool> InternalizeSymbols(
454 "amdgpu-internalize-symbols",
455 cl::desc("Enable elimination of non-kernel functions and unused globals"),
456 cl::init(Val: false),
457 cl::Hidden);
458
459// Option to inline all early.
460static cl::opt<bool> EarlyInlineAll(
461 "amdgpu-early-inline-all",
462 cl::desc("Inline all functions early"),
463 cl::init(Val: false),
464 cl::Hidden);
465
466static cl::opt<bool> RemoveIncompatibleFunctions(
467 "amdgpu-enable-remove-incompatible-functions", cl::Hidden,
468 cl::desc("Enable removal of functions when they"
469 "use features not supported by the target GPU"),
470 cl::init(Val: true));
471
472static cl::opt<bool> EnableSDWAPeephole(
473 "amdgpu-sdwa-peephole",
474 cl::desc("Enable SDWA peepholer"),
475 cl::init(Val: true));
476
477static cl::opt<bool> EnableDPPCombine(
478 "amdgpu-dpp-combine",
479 cl::desc("Enable DPP combiner"),
480 cl::init(Val: true));
481
482// Enable address space based alias analysis
483static cl::opt<bool> EnableAMDGPUAliasAnalysis("enable-amdgpu-aa", cl::Hidden,
484 cl::desc("Enable AMDGPU Alias Analysis"),
485 cl::init(Val: true));
486
487static cl::opt<bool>
488 XnackSetting("amdgpu-xnack",
489 cl::desc("Force amdgpu.xnack value for testing"),
490 cl::ReallyHidden);
491
492static cl::opt<bool>
493 SramEccSetting("amdgpu-sramecc",
494 cl::desc("Force amdgpu.sramecc for testing"),
495 cl::ReallyHidden);
496
497// Enable lib calls simplifications
498static cl::opt<bool> EnableLibCallSimplify(
499 "amdgpu-simplify-libcall",
500 cl::desc("Enable amdgpu library simplifications"),
501 cl::init(Val: true),
502 cl::Hidden);
503
504static cl::opt<bool> EnableLowerKernelArguments(
505 "amdgpu-ir-lower-kernel-arguments",
506 cl::desc("Lower kernel argument loads in IR pass"),
507 cl::init(Val: true),
508 cl::Hidden);
509
510static cl::opt<bool> EnableRegReassign(
511 "amdgpu-reassign-regs",
512 cl::desc("Enable register reassign optimizations on gfx10+"),
513 cl::init(Val: true),
514 cl::Hidden);
515
516static cl::opt<bool> OptVGPRLiveRange(
517 "amdgpu-opt-vgpr-liverange",
518 cl::desc("Enable VGPR liverange optimizations for if-else structure"),
519 cl::init(Val: true), cl::Hidden);
520
521static cl::opt<ScanOptions> AMDGPUAtomicOptimizerStrategy(
522 "amdgpu-atomic-optimizer-strategy",
523 cl::desc("Select DPP or Iterative strategy for scan"),
524 cl::init(Val: ScanOptions::Iterative),
525 cl::values(
526 clEnumValN(ScanOptions::DPP, "DPP", "Use DPP operations for scan"),
527 clEnumValN(ScanOptions::Iterative, "Iterative",
528 "Use Iterative approach for scan"),
529 clEnumValN(ScanOptions::None, "None", "Disable atomic optimizer")));
530
531// Enable Mode register optimization
532static cl::opt<bool> EnableSIModeRegisterPass(
533 "amdgpu-mode-register",
534 cl::desc("Enable mode register pass"),
535 cl::init(Val: true),
536 cl::Hidden);
537
538// Enable GFX11+ s_delay_alu insertion
539static cl::opt<bool>
540 EnableInsertDelayAlu("amdgpu-enable-delay-alu",
541 cl::desc("Enable s_delay_alu insertion"),
542 cl::init(Val: true), cl::Hidden);
543
544// Enable GFX11+ VOPD
545static cl::opt<bool>
546 EnableVOPD("amdgpu-enable-vopd",
547 cl::desc("Enable VOPD, dual issue of VALU in wave32"),
548 cl::init(Val: true), cl::Hidden);
549
550// Option is used in lit tests to prevent deadcoding of patterns inspected.
551static cl::opt<bool>
552EnableDCEInRA("amdgpu-dce-in-ra",
553 cl::init(Val: true), cl::Hidden,
554 cl::desc("Enable machine DCE inside regalloc"));
555
556static cl::opt<bool> EnableSetWavePriority("amdgpu-set-wave-priority",
557 cl::desc("Adjust wave priority"),
558 cl::init(Val: false), cl::Hidden);
559
560static cl::opt<bool> EnableScalarIRPasses(
561 "amdgpu-scalar-ir-passes",
562 cl::desc("Enable scalar IR passes"),
563 cl::init(Val: true),
564 cl::Hidden);
565
566static cl::opt<bool> EnableLowerExecSync(
567 "amdgpu-enable-lower-exec-sync",
568 cl::desc("Enable lowering of execution synchronization."), cl::init(Val: true),
569 cl::Hidden);
570
571static cl::opt<bool>
572 EnableSwLowerLDS("amdgpu-enable-sw-lower-lds",
573 cl::desc("Enable lowering of lds to global memory pass "
574 "and asan instrument resulting IR."),
575 cl::init(Val: true), cl::Hidden);
576
577static cl::opt<bool, true> EnableObjectLinking(
578 "amdgpu-enable-object-linking",
579 cl::desc("Enable object linking for cross-TU LDS and ABI support"),
580 cl::location(L&: AMDGPUTargetMachine::EnableObjectLinking), cl::init(Val: false),
581 cl::Hidden);
582
583static cl::opt<bool, true> EnableLowerModuleLDS(
584 "amdgpu-enable-lower-module-lds", cl::desc("Enable lower module lds pass"),
585 cl::location(L&: AMDGPUTargetMachine::EnableLowerModuleLDS), cl::init(Val: true),
586 cl::Hidden);
587
588static cl::opt<bool> EnablePreRAOptimizations(
589 "amdgpu-enable-pre-ra-optimizations",
590 cl::desc("Enable Pre-RA optimizations pass"), cl::init(Val: true),
591 cl::Hidden);
592
593static cl::opt<bool> EnablePromoteKernelArguments(
594 "amdgpu-enable-promote-kernel-arguments",
595 cl::desc("Enable promotion of flat kernel pointer arguments to global"),
596 cl::Hidden, cl::init(Val: true));
597
598static cl::opt<bool> EnableImageIntrinsicOptimizer(
599 "amdgpu-enable-image-intrinsic-optimizer",
600 cl::desc("Enable image intrinsic optimizer pass"), cl::init(Val: true),
601 cl::Hidden);
602
603static cl::opt<bool>
604 EnableLoopPrefetch("amdgpu-loop-prefetch",
605 cl::desc("Enable loop data prefetch on AMDGPU"),
606 cl::Hidden, cl::init(Val: false));
607
608static cl::opt<std::string>
609 AMDGPUSchedStrategy("amdgpu-sched-strategy",
610 cl::desc("Select custom AMDGPU scheduling strategy."),
611 cl::Hidden, cl::init(Val: ""));
612
613// Scheduler selection is consulted both when creating the scheduler and from
614// overrideSchedPolicy(), so keep the attribute and global command line handling
615// in one helper.
616StringRef llvm::AMDGPU::getSchedStrategy(const Function &F) {
617 Attribute SchedStrategyAttr = F.getFnAttribute(Kind: "amdgpu-sched-strategy");
618 if (SchedStrategyAttr.isValid())
619 return SchedStrategyAttr.getValueAsString();
620
621 if (!AMDGPUSchedStrategy.empty())
622 return AMDGPUSchedStrategy;
623
624 return "";
625}
626
627static void
628diagnoseUnsupportedCoExecSchedulerSelection(const Function &F,
629 const GCNSubtarget &ST) {
630 if (ST.hasGFX1250Insts())
631 return;
632
633 F.getContext().diagnose(DI: DiagnosticInfoUnsupported(
634 F, "'amdgpu-sched-strategy'='coexec' is only supported for gfx1250",
635 DiagnosticLocation(), DS_Warning));
636}
637
638static bool useNoopPostScheduler(const Function &F) {
639 Attribute PostSchedStrategyAttr =
640 F.getFnAttribute(Kind: "amdgpu-post-sched-strategy");
641 return PostSchedStrategyAttr.isValid() &&
642 PostSchedStrategyAttr.getValueAsString() == "nop";
643}
644
645static cl::opt<bool> EnableRewritePartialRegUses(
646 "amdgpu-enable-rewrite-partial-reg-uses",
647 cl::desc("Enable rewrite partial reg uses pass"), cl::init(Val: true),
648 cl::Hidden);
649
650static cl::opt<bool> EnableHipStdPar(
651 "amdgpu-enable-hipstdpar",
652 cl::desc("Enable HIP Standard Parallelism Offload support"), cl::init(Val: false),
653 cl::Hidden);
654
655static cl::opt<bool>
656 EnableAMDGPUAttributor("amdgpu-attributor-enable",
657 cl::desc("Enable AMDGPUAttributorPass"),
658 cl::init(Val: true), cl::Hidden);
659
660static cl::opt<bool> HasClosedWorldAssumption(
661 "amdgpu-link-time-closed-world",
662 cl::desc("Whether has closed-world assumption at link time"),
663 cl::init(Val: false), cl::Hidden);
664
665static cl::opt<bool> EnableUniformIntrinsicCombine(
666 "amdgpu-enable-uniform-intrinsic-combine",
667 cl::desc("Enable/Disable the Uniform Intrinsic Combine Pass"),
668 cl::init(Val: true), cl::Hidden);
669
670static cl::opt<bool>
671 EnableMachinePipeliner("amdgpu-enable-pipeliner",
672 cl::desc("Enable Machine Pipeliner for AMDGCN"),
673 cl::init(Val: false), cl::Hidden);
674
675extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
676 // Register the target
677 RegisterTargetMachine<R600TargetMachine> X(getTheR600Target());
678 RegisterTargetMachine<GCNTargetMachine> Y(getTheGCNTarget());
679 RegisterTargetMachine<GCNTargetMachine> YLegacy(getTheGCNLegacyTarget());
680
681 PassRegistry *PR = PassRegistry::getPassRegistry();
682 initializeR600ClauseMergePassPass(*PR);
683 initializeR600ControlFlowFinalizerPass(*PR);
684 initializeR600PacketizerPass(*PR);
685 initializeR600ExpandSpecialInstrsPassPass(*PR);
686 initializeR600VectorRegMergerPass(*PR);
687 initializeR600EmitClauseMarkersPass(*PR);
688 initializeR600MachineCFGStructurizerPass(*PR);
689 initializeGlobalISel(*PR);
690 initializeAMDGPUAsmPrinterPass(*PR);
691 initializeAMDGPUDAGToDAGISelLegacyPass(*PR);
692 initializeAMDGPUPrepareAGPRAllocLegacyPass(*PR);
693 initializeGCNDPPCombineLegacyPass(*PR);
694 initializeSILowerI1CopiesLegacyPass(*PR);
695 initializeAMDGPUGlobalISelDivergenceLoweringLegacyPass(*PR);
696 initializeAMDGPURegBankSelectPass(*PR);
697 initializeAMDGPURegBankLegalizePass(*PR);
698 initializeSILowerWWMCopiesLegacyPass(*PR);
699 initializeAMDGPUMarkLastScratchLoadLegacyPass(*PR);
700 initializeSILowerSGPRSpillsLegacyPass(*PR);
701 initializeSIFixSGPRCopiesLegacyPass(*PR);
702 initializeSIFixVGPRCopiesLegacyPass(*PR);
703 initializeSIFoldOperandsLegacyPass(*PR);
704 initializeSIPeepholeSDWALegacyPass(*PR);
705 initializeSIShrinkInstructionsLegacyPass(*PR);
706 initializeSIOptimizeExecMaskingPreRALegacyPass(*PR);
707 initializeSIOptimizeVGPRLiveRangeLegacyPass(*PR);
708 initializeAMDGPUNextUseAnalysisLegacyPassPass(*PR);
709 initializeAMDGPUNextUseAnalysisPrinterLegacyPassPass(*PR);
710 initializeSILoadStoreOptimizerLegacyPass(*PR);
711 initializeAMDGPUCtorDtorLoweringLegacyPass(*PR);
712 initializeAMDGPUAlwaysInlinePass(*PR);
713 initializeAMDGPULowerExecSyncLegacyPass(*PR);
714 initializeAMDGPUSwLowerLDSLegacyPass(*PR);
715 initializeAMDGPUAnnotateUniformValuesLegacyPass(*PR);
716 initializeAMDGPUAtomicOptimizerPass(*PR);
717 initializeAMDGPULowerKernelArgumentsPass(*PR);
718 initializeAMDGPUPromoteKernelArgumentsPass(*PR);
719 initializeAMDGPULowerKernelAttributesPass(*PR);
720 initializeAMDGPUExportKernelRuntimeHandlesLegacyPass(*PR);
721 initializeAMDGPUPostLegalizerCombinerPass(*PR);
722 initializeAMDGPUPreLegalizerCombinerPass(*PR);
723 initializeAMDGPURegBankCombinerPass(*PR);
724 initializeAMDGPUPromoteAllocaPass(*PR);
725 initializeAMDGPUCodeGenPreparePass(*PR);
726 initializeAMDGPULateCodeGenPrepareLegacyPass(*PR);
727 initializeAMDGPURemoveIncompatibleFunctionsLegacyPass(*PR);
728 initializeAMDGPULowerModuleLDSLegacyPass(*PR);
729 initializeAMDGPULowerBufferFatPointersPass(*PR);
730 initializeAMDGPULowerIntrinsicsLegacyPass(*PR);
731 initializeAMDGPUReserveWWMRegsLegacyPass(*PR);
732 initializeAMDGPURewriteAGPRCopyMFMALegacyPass(*PR);
733 initializeAMDGPURewriteOutArgumentsPass(*PR);
734 initializeAMDGPURewriteUndefForPHILegacyPass(*PR);
735 initializeSIAnnotateControlFlowLegacyPass(*PR);
736 initializeAMDGPUInsertDelayAluLegacyPass(*PR);
737 initializeAMDGPULowerVGPREncodingLegacyPass(*PR);
738 initializeSIInsertHardClausesLegacyPass(*PR);
739 initializeSIInsertWaitcntsLegacyPass(*PR);
740 initializeSIModeRegisterLegacyPass(*PR);
741 initializeSIWholeQuadModeLegacyPass(*PR);
742 initializeSILowerControlFlowLegacyPass(*PR);
743 initializeSIPreEmitPeepholeLegacyPass(*PR);
744 initializeSILateBranchLoweringLegacyPass(*PR);
745 initializeSIMemoryLegalizerLegacyPass(*PR);
746 initializeSIOptimizeExecMaskingLegacyPass(*PR);
747 initializeSIPreAllocateWWMRegsLegacyPass(*PR);
748 initializeSIFormMemoryClausesLegacyPass(*PR);
749 initializeSIPostRABundlerLegacyPass(*PR);
750 initializeGCNCreateVOPDLegacyPass(*PR);
751 initializeAMDGPUUnifyDivergentExitNodesLegacyPass(*PR);
752 initializeAMDGPUAAWrapperPassPass(*PR);
753 initializeAMDGPUExternalAAWrapperPass(*PR);
754 initializeAMDGPUImageIntrinsicOptimizerPass(*PR);
755 initializeAMDGPUPrintfRuntimeBindingPass(*PR);
756 initializeAMDGPUResourceUsageAnalysisWrapperPassPass(*PR);
757 initializeGCNNSAReassignLegacyPass(*PR);
758 initializeGCNPreRAOptimizationsLegacyPass(*PR);
759 initializeGCNPreRALongBranchRegLegacyPass(*PR);
760 initializeGCNRewritePartialRegUsesLegacyPass(*PR);
761 initializeGCNRegPressurePrinterPass(*PR);
762 initializeAMDGPUPreloadKernArgPrologLegacyPass(*PR);
763 initializeAMDGPUWaitSGPRHazardsLegacyPass(*PR);
764 initializeAMDGPUPreloadKernelArgumentsLegacyPass(*PR);
765 initializeAMDGPUUniformIntrinsicCombineLegacyPass(*PR);
766}
767
768static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
769 return std::make_unique<AMDGPUTargetObjectFile>();
770}
771
772static ScheduleDAGInstrs *createSIMachineScheduler(MachineSchedContext *C) {
773 return new SIScheduleDAGMI(C);
774}
775
776static ScheduleDAGInstrs *
777createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
778 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
779 ScheduleDAGMILive *DAG =
780 new GCNScheduleDAGMILive(C, std::make_unique<GCNMaxOccupancySchedStrategy>(args&: C));
781 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
782 if (ST.shouldClusterStores())
783 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
784 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
785 DAG->addMutation(Mutation: createAMDGPUMacroFusionDAGMutation());
786 DAG->addMutation(Mutation: createAMDGPUExportClusteringDAGMutation());
787 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
788 DAG->addMutation(Mutation: createAMDGPUHazardLatencyDAGMutation(MF: C->MF));
789 return DAG;
790}
791
792static ScheduleDAGInstrs *
793createGCNMaxILPMachineScheduler(MachineSchedContext *C) {
794 ScheduleDAGMILive *DAG =
795 new GCNScheduleDAGMILive(C, std::make_unique<GCNMaxILPSchedStrategy>(args&: C));
796 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
797 return DAG;
798}
799
800static ScheduleDAGInstrs *
801createGCNMaxMemoryClauseMachineScheduler(MachineSchedContext *C) {
802 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
803 ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive(
804 C, std::make_unique<GCNMaxMemoryClauseSchedStrategy>(args&: C));
805 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
806 if (ST.shouldClusterStores())
807 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
808 DAG->addMutation(Mutation: createAMDGPUExportClusteringDAGMutation());
809 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
810 DAG->addMutation(Mutation: createAMDGPUHazardLatencyDAGMutation(MF: C->MF));
811 return DAG;
812}
813
814static ScheduleDAGInstrs *
815createIterativeGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
816 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
817 auto *DAG = new GCNIterativeScheduler(
818 C, GCNIterativeScheduler::SCHEDULE_LEGACYMAXOCCUPANCY);
819 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
820 if (ST.shouldClusterStores())
821 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
822 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
823 return DAG;
824}
825
826static ScheduleDAGInstrs *createMinRegScheduler(MachineSchedContext *C) {
827 auto *DAG = new GCNIterativeScheduler(
828 C, GCNIterativeScheduler::SCHEDULE_MINREGFORCED);
829 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
830 return DAG;
831}
832
833static ScheduleDAGInstrs *
834createIterativeILPMachineScheduler(MachineSchedContext *C) {
835 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
836 auto *DAG = new GCNIterativeScheduler(C, GCNIterativeScheduler::SCHEDULE_ILP);
837 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
838 if (ST.shouldClusterStores())
839 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
840 DAG->addMutation(Mutation: createAMDGPUMacroFusionDAGMutation());
841 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
842 return DAG;
843}
844
845static MachineSchedRegistry
846SISchedRegistry("si", "Run SI's custom scheduler",
847 createSIMachineScheduler);
848
849static MachineSchedRegistry
850GCNMaxOccupancySchedRegistry("gcn-max-occupancy",
851 "Run GCN scheduler to maximize occupancy",
852 createGCNMaxOccupancyMachineScheduler);
853
854static MachineSchedRegistry
855 GCNMaxILPSchedRegistry("gcn-max-ilp", "Run GCN scheduler to maximize ilp",
856 createGCNMaxILPMachineScheduler);
857
858static MachineSchedRegistry GCNMaxMemoryClauseSchedRegistry(
859 "gcn-max-memory-clause", "Run GCN scheduler to maximize memory clause",
860 createGCNMaxMemoryClauseMachineScheduler);
861
862static MachineSchedRegistry IterativeGCNMaxOccupancySchedRegistry(
863 "gcn-iterative-max-occupancy-experimental",
864 "Run GCN scheduler to maximize occupancy (experimental)",
865 createIterativeGCNMaxOccupancyMachineScheduler);
866
867static MachineSchedRegistry GCNMinRegSchedRegistry(
868 "gcn-iterative-minreg",
869 "Run GCN iterative scheduler for minimal register usage (experimental)",
870 createMinRegScheduler);
871
872static MachineSchedRegistry GCNILPSchedRegistry(
873 "gcn-iterative-ilp",
874 "Run GCN iterative scheduler for ILP scheduling (experimental)",
875 createIterativeILPMachineScheduler);
876
877LLVM_READNONE
878static StringRef getGPUOrDefault(const Triple &TT, StringRef GPU) {
879 if (!GPU.empty())
880 return GPU;
881
882 if (StringRef Name = AMDGPU::getArchNameFromSubArch(SubArch: TT.getSubArch());
883 !Name.empty())
884 return Name;
885
886 // Need to default to a target with flat support for HSA.
887 if (TT.isAMDGCN())
888 return TT.getOS() == Triple::AMDHSA ? "generic-hsa" : "generic";
889
890 return "r600";
891}
892
893static Reloc::Model getEffectiveRelocModel() {
894 // The AMDGPU toolchain only supports generating shared objects, so we
895 // must always use PIC.
896 return Reloc::PIC_;
897}
898
899AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT,
900 StringRef CPU, StringRef FS,
901 const TargetOptions &Options,
902 std::optional<Reloc::Model> RM,
903 std::optional<CodeModel::Model> CM,
904 CodeGenOptLevel OptLevel)
905 : CodeGenTargetMachineImpl(
906 T, TT.computeDataLayout(), TT, getGPUOrDefault(TT, GPU: CPU), FS, Options,
907 getEffectiveRelocModel(), getEffectiveCodeModel(CM, Default: CodeModel::Small),
908 OptLevel),
909 TLOF(createTLOF(TT: getTargetTriple())) {
910 initAsmInfo();
911 if (TT.isAMDGCN()) {
912 // Triple is missing a representation for non-empty, but unrecognized
913 // subarches. Only permit no subarch for any subtarget if it was really
914 // empty.
915 bool IsUnknownSubArch =
916 TT.getSubArch() == Triple::NoSubArch && TT.getArchName().size() != 6;
917 if (IsUnknownSubArch)
918 reportFatalUsageError(reason: "unknown subarch " + TT.getArchName());
919
920 if (TT.getSubArch() != Triple::NoSubArch) {
921 AMDGPU::GPUKind Kind = AMDGPU::parseArchAMDGCN(CPU);
922 Triple::SubArchType GPUSubArch = AMDGPU::getSubArch(AK: Kind);
923 if (Kind != AMDGPU::GK_NONE && GPUSubArch != TT.getSubArch() &&
924 TT.getSubArch() != AMDGPU::getMajorSubArch(SubArch: GPUSubArch)) {
925 reportFatalUsageError(reason: "invalid cpu '" + CPU + "' for subarch " +
926 TT.getArchName());
927 }
928 }
929
930 if (getMCSubtargetInfo().checkFeatures(FS: "+wavefrontsize64"))
931 MRI.reset(p: llvm::createGCNMCRegisterInfo(DwarfFlavour: AMDGPUDwarfFlavour::Wave64));
932 else if (getMCSubtargetInfo().checkFeatures(FS: "+wavefrontsize32"))
933 MRI.reset(p: llvm::createGCNMCRegisterInfo(DwarfFlavour: AMDGPUDwarfFlavour::Wave32));
934 }
935 LLT::setUseExtended(true);
936}
937
938bool AMDGPUTargetMachine::EnableFunctionCalls = false;
939bool AMDGPUTargetMachine::EnableObjectLinking = false;
940bool AMDGPUTargetMachine::EnableLowerModuleLDS = true;
941
942AMDGPUTargetMachine::~AMDGPUTargetMachine() = default;
943
944StringRef AMDGPUTargetMachine::getGPUName(const Function &F) const {
945 Attribute GPUAttr = F.getFnAttribute(Kind: "target-cpu");
946 return GPUAttr.isValid() ? GPUAttr.getValueAsString() : getTargetCPU();
947}
948
949StringRef AMDGPUTargetMachine::getFeatureString(const Function &F) const {
950 Attribute FSAttr = F.getFnAttribute(Kind: "target-features");
951
952 return FSAttr.isValid() ? FSAttr.getValueAsString()
953 : getTargetFeatureString();
954}
955
956llvm::ScheduleDAGInstrs *
957AMDGPUTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
958 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
959 ScheduleDAGMILive *DAG = createSchedLive(C);
960 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
961 if (ST.shouldClusterStores())
962 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
963 return DAG;
964}
965
966/// Predicate for Internalize pass.
967static bool mustPreserveGV(const GlobalValue &GV) {
968 if (const Function *F = dyn_cast<Function>(Val: &GV))
969 return F->isDeclaration() || F->getName().starts_with(Prefix: "__asan_") ||
970 F->getName().starts_with(Prefix: "__sanitizer_") ||
971 AMDGPU::isEntryFunctionCC(CC: F->getCallingConv());
972
973 GV.removeDeadConstantUsers();
974 return !GV.use_empty();
975}
976
977void AMDGPUTargetMachine::registerDefaultAliasAnalyses(AAManager &AAM) {
978 if (EnableAMDGPUAliasAnalysis)
979 AAM.registerFunctionAnalysis<AMDGPUAA>();
980}
981
982static Expected<ScanOptions>
983parseAMDGPUAtomicOptimizerStrategy(StringRef Params) {
984 if (Params.empty())
985 return ScanOptions::Iterative;
986 Params.consume_front(Prefix: "strategy=");
987 auto Result = StringSwitch<std::optional<ScanOptions>>(Params)
988 .Case(S: "dpp", Value: ScanOptions::DPP)
989 .Cases(CaseStrings: {"iterative", ""}, Value: ScanOptions::Iterative)
990 .Case(S: "none", Value: ScanOptions::None)
991 .Default(Value: std::nullopt);
992 if (Result)
993 return *Result;
994 return make_error<StringError>(Args: "invalid parameter", Args: inconvertibleErrorCode());
995}
996
997Expected<AMDGPUAttributorOptions>
998parseAMDGPUAttributorPassOptions(StringRef Params) {
999 AMDGPUAttributorOptions Result;
1000 while (!Params.empty()) {
1001 StringRef ParamName;
1002 std::tie(args&: ParamName, args&: Params) = Params.split(Separator: ';');
1003 if (ParamName == "closed-world") {
1004 Result.IsClosedWorld = true;
1005 } else {
1006 return make_error<StringError>(
1007 Args: formatv(Fmt: "invalid AMDGPUAttributor pass parameter '{0}' ", Vals&: ParamName)
1008 .str(),
1009 Args: inconvertibleErrorCode());
1010 }
1011 }
1012 return Result;
1013}
1014
1015void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
1016
1017#define GET_PASS_REGISTRY "AMDGPUPassRegistry.def"
1018#include "llvm/Passes/TargetPassRegistry.inc"
1019
1020 PB.registerPipelineParsingCallback(
1021 C: [this](StringRef Name, CGSCCPassManager &PM,
1022 ArrayRef<PassBuilder::PipelineElement> Pipeline) {
1023 if (Name == "amdgpu-attributor-cgscc" && getTargetTriple().isAMDGCN()) {
1024 PM.addPass(Pass: AMDGPUAttributorCGSCCPass(
1025 *static_cast<GCNTargetMachine *>(this)));
1026 return true;
1027 }
1028 return false;
1029 });
1030
1031 PB.registerScalarOptimizerLateEPCallback(
1032 C: [](FunctionPassManager &FPM, OptimizationLevel Level) {
1033 if (Level == OptimizationLevel::O0)
1034 return;
1035
1036 FPM.addPass(Pass: InferAddressSpacesPass());
1037 });
1038
1039 PB.registerVectorizerEndEPCallback(
1040 C: [](FunctionPassManager &FPM, OptimizationLevel Level) {
1041 if (Level == OptimizationLevel::O0)
1042 return;
1043
1044 FPM.addPass(Pass: InferAddressSpacesPass());
1045 });
1046
1047 PB.registerPipelineEarlySimplificationEPCallback(
1048 C: [this](ModulePassManager &PM, OptimizationLevel Level,
1049 ThinOrFullLTOPhase Phase) {
1050 if (!isLTOPreLink(Phase) && getTargetTriple().isAMDGCN()) {
1051 // When we are not using -fgpu-rdc, we can run accelerator code
1052 // selection relatively early, but still after linking to prevent
1053 // eager removal of potentially reachable symbols.
1054 if (EnableHipStdPar) {
1055 PM.addPass(Pass: HipStdParMathFixupPass());
1056 PM.addPass(Pass: HipStdParAcceleratorCodeSelectionPass());
1057 }
1058
1059 PM.addPass(Pass: AMDGPUPrintfRuntimeBindingPass());
1060 }
1061
1062 if (Level == OptimizationLevel::O0)
1063 return;
1064
1065 // We don't want to run internalization at per-module stage.
1066 if (InternalizeSymbols && !isLTOPreLink(Phase)) {
1067 PM.addPass(Pass: InternalizePass(mustPreserveGV));
1068 PM.addPass(Pass: GlobalDCEPass());
1069 }
1070
1071 if (EarlyInlineAll && !EnableFunctionCalls)
1072 PM.addPass(Pass: AMDGPUAlwaysInlinePass());
1073 });
1074
1075 PB.registerPeepholeEPCallback(
1076 C: [](FunctionPassManager &FPM, OptimizationLevel Level) {
1077 if (Level == OptimizationLevel::O0)
1078 return;
1079
1080 FPM.addPass(Pass: AMDGPUUseNativeCallsPass());
1081 if (EnableLibCallSimplify)
1082 FPM.addPass(Pass: AMDGPUSimplifyLibCallsPass());
1083
1084 if (EnableUniformIntrinsicCombine)
1085 FPM.addPass(Pass: AMDGPUUniformIntrinsicCombinePass());
1086 });
1087
1088 PB.registerCGSCCOptimizerLateEPCallback(
1089 C: [this](CGSCCPassManager &PM, OptimizationLevel Level) {
1090 if (Level == OptimizationLevel::O0)
1091 return;
1092
1093 FunctionPassManager FPM;
1094
1095 // Add promote kernel arguments pass to the opt pipeline right before
1096 // infer address spaces which is needed to do actual address space
1097 // rewriting.
1098 if (Level > OptimizationLevel::O1 && EnablePromoteKernelArguments)
1099 FPM.addPass(Pass: AMDGPUPromoteKernelArgumentsPass());
1100
1101 // Add infer address spaces pass to the opt pipeline after inlining
1102 // but before SROA to increase SROA opportunities.
1103 FPM.addPass(Pass: InferAddressSpacesPass());
1104
1105 // This should run after inlining to have any chance of doing
1106 // anything, and before other cleanup optimizations.
1107 FPM.addPass(Pass: AMDGPULowerKernelAttributesPass());
1108
1109 // Promote alloca to vector before SROA and loop unroll. If we
1110 // manage to eliminate allocas before unroll we may choose to unroll
1111 // less.
1112 FPM.addPass(Pass: AMDGPUPromoteAllocaToVectorPass(*this));
1113
1114 PM.addPass(Pass: createCGSCCToFunctionPassAdaptor(Pass: std::move(FPM)));
1115 });
1116
1117 // FIXME: Why is AMDGPUAttributor not in CGSCC?
1118 PB.registerOptimizerLastEPCallback(C: [this](ModulePassManager &MPM,
1119 OptimizationLevel Level,
1120 ThinOrFullLTOPhase Phase) {
1121 if (Level != OptimizationLevel::O0) {
1122 if (!isLTOPreLink(Phase)) {
1123 if (EnableAMDGPUAttributor && getTargetTriple().isAMDGCN()) {
1124 AMDGPUAttributorOptions Opts;
1125 MPM.addPass(Pass: AMDGPUAttributorPass(*this, Opts, Phase));
1126 }
1127 }
1128 }
1129 });
1130
1131 PB.registerFullLinkTimeOptimizationLastEPCallback(
1132 C: [this](ModulePassManager &PM, OptimizationLevel Level) {
1133 // Clean up redundant memory round-trips that the full-LTO pipeline,
1134 // unlike the non-LTO/ThinLTO ones, otherwise leaves for codegen.
1135 if (Level != OptimizationLevel::O0) {
1136 PM.addPass(Pass: createModuleToFunctionPassAdaptor(
1137 Pass: EarlyCSEPass(/*UseMemorySSA=*/true)));
1138 }
1139
1140 // When we are using -fgpu-rdc, we can only run accelerator code
1141 // selection after linking to prevent, otherwise we end up removing
1142 // potentially reachable symbols that were exported as external in other
1143 // modules.
1144 if (EnableHipStdPar) {
1145 PM.addPass(Pass: HipStdParMathFixupPass());
1146 PM.addPass(Pass: HipStdParAcceleratorCodeSelectionPass());
1147 }
1148 // We want to support the -lto-partitions=N option as "best effort".
1149 // For that, we need to lower LDS earlier in the pipeline before the
1150 // module is partitioned for codegen.
1151 if (EnableLowerExecSync)
1152 PM.addPass(Pass: AMDGPULowerExecSyncPass());
1153 if (EnableSwLowerLDS)
1154 PM.addPass(Pass: AMDGPUSwLowerLDSPass());
1155 if (EnableLowerModuleLDS)
1156 PM.addPass(Pass: AMDGPULowerModuleLDSPass(*this));
1157 if (Level != OptimizationLevel::O0) {
1158 // We only want to run this with O2 or higher since inliner and SROA
1159 // don't run in O1.
1160 if (Level != OptimizationLevel::O1) {
1161 PM.addPass(
1162 Pass: createModuleToFunctionPassAdaptor(Pass: InferAddressSpacesPass()));
1163 }
1164 // Do we really need internalization in LTO?
1165 if (InternalizeSymbols) {
1166 PM.addPass(Pass: InternalizePass(mustPreserveGV));
1167 PM.addPass(Pass: GlobalDCEPass());
1168 }
1169 if (EnableAMDGPUAttributor && getTargetTriple().isAMDGCN()) {
1170 AMDGPUAttributorOptions Opt;
1171 if (HasClosedWorldAssumption)
1172 Opt.IsClosedWorld = true;
1173 PM.addPass(Pass: AMDGPUAttributorPass(
1174 *this, Opt, ThinOrFullLTOPhase::FullLTOPostLink));
1175 }
1176 }
1177 if (!NoKernelInfoEndLTO) {
1178 FunctionPassManager FPM;
1179 FPM.addPass(Pass: KernelInfoPrinter(this));
1180 PM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
1181 }
1182 });
1183
1184 PB.registerRegClassFilterParsingCallback(
1185 C: [](StringRef FilterName) -> RegAllocFilterFunc {
1186 if (FilterName == "sgpr")
1187 return onlyAllocateSGPRs;
1188 if (FilterName == "vgpr")
1189 return onlyAllocateVGPRs;
1190 if (FilterName == "wwm")
1191 return onlyAllocateWWMRegs;
1192 return nullptr;
1193 });
1194}
1195
1196bool AMDGPUTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
1197 unsigned DestAS) const {
1198 return AMDGPU::isFlatGlobalAddrSpace(AS: SrcAS) &&
1199 AMDGPU::isFlatGlobalAddrSpace(AS: DestAS);
1200}
1201
1202unsigned AMDGPUTargetMachine::getAssumedAddrSpace(const Value *V) const {
1203 if (auto *Arg = dyn_cast<Argument>(Val: V);
1204 Arg &&
1205 AMDGPU::isModuleEntryFunctionCC(CC: Arg->getParent()->getCallingConv()) &&
1206 !Arg->hasByRefAttr())
1207 return AMDGPUAS::GLOBAL_ADDRESS;
1208
1209 const auto *LD = dyn_cast<LoadInst>(Val: V);
1210 if (!LD) // TODO: Handle invariant load like constant.
1211 return AMDGPUAS::UNKNOWN_ADDRESS_SPACE;
1212
1213 // It must be a generic pointer loaded.
1214 assert(V->getType()->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS);
1215
1216 const auto *Ptr = LD->getPointerOperand();
1217 if (Ptr->getType()->getPointerAddressSpace() != AMDGPUAS::CONSTANT_ADDRESS)
1218 return AMDGPUAS::UNKNOWN_ADDRESS_SPACE;
1219 // For a generic pointer loaded from the constant memory, it could be assumed
1220 // as a global pointer since the constant memory is only populated on the
1221 // host side. As implied by the offload programming model, only global
1222 // pointers could be referenced on the host side.
1223 return AMDGPUAS::GLOBAL_ADDRESS;
1224}
1225
1226std::pair<const Value *, unsigned>
1227AMDGPUTargetMachine::getPredicatedAddrSpace(const Value *V) const {
1228 if (auto *II = dyn_cast<IntrinsicInst>(Val: V)) {
1229 switch (II->getIntrinsicID()) {
1230 case Intrinsic::amdgcn_is_shared:
1231 return std::pair(II->getArgOperand(i: 0), AMDGPUAS::LOCAL_ADDRESS);
1232 case Intrinsic::amdgcn_is_private:
1233 return std::pair(II->getArgOperand(i: 0), AMDGPUAS::PRIVATE_ADDRESS);
1234 default:
1235 break;
1236 }
1237 return std::pair(nullptr, -1);
1238 }
1239 // Check the global pointer predication based on
1240 // (!is_share(p) && !is_private(p)). Note that logic 'and' is commutative and
1241 // the order of 'is_shared' and 'is_private' is not significant.
1242 Value *Ptr;
1243 if (match(
1244 V: const_cast<Value *>(V),
1245 P: m_c_And(L: m_Not(V: m_Intrinsic<Intrinsic::amdgcn_is_shared>(Ops: m_Value(V&: Ptr))),
1246 R: m_Not(V: m_Intrinsic<Intrinsic::amdgcn_is_private>(
1247 Ops: m_Deferred(V: Ptr))))))
1248 return std::pair(Ptr, AMDGPUAS::GLOBAL_ADDRESS);
1249
1250 return std::pair(nullptr, -1);
1251}
1252
1253unsigned
1254AMDGPUTargetMachine::getAddressSpaceForPseudoSourceKind(unsigned Kind) const {
1255 switch (Kind) {
1256 case PseudoSourceValue::Stack:
1257 case PseudoSourceValue::FixedStack:
1258 return AMDGPUAS::PRIVATE_ADDRESS;
1259 case PseudoSourceValue::ConstantPool:
1260 case PseudoSourceValue::GOT:
1261 case PseudoSourceValue::JumpTable:
1262 case PseudoSourceValue::GlobalValueCallEntry:
1263 case PseudoSourceValue::ExternalSymbolCallEntry:
1264 return AMDGPUAS::CONSTANT_ADDRESS;
1265 }
1266 return AMDGPUAS::FLAT_ADDRESS;
1267}
1268
1269bool AMDGPUTargetMachine::splitModule(
1270 Module &M, unsigned NumParts,
1271 function_ref<void(std::unique_ptr<Module> MPart)> ModuleCallback) {
1272 // FIXME(?): Would be better to use an already existing Analysis/PassManager,
1273 // but all current users of this API don't have one ready and would need to
1274 // create one anyway. Let's hide the boilerplate for now to keep it simple.
1275
1276 LoopAnalysisManager LAM;
1277 FunctionAnalysisManager FAM;
1278 CGSCCAnalysisManager CGAM;
1279 ModuleAnalysisManager MAM;
1280
1281 PassBuilder PB(this);
1282 PB.registerModuleAnalyses(MAM);
1283 PB.registerFunctionAnalyses(FAM);
1284 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
1285
1286 ModulePassManager MPM;
1287 MPM.addPass(Pass: AMDGPUSplitModulePass(NumParts, ModuleCallback));
1288 MPM.run(IR&: M, AM&: MAM);
1289 return true;
1290}
1291
1292//===----------------------------------------------------------------------===//
1293// GCN Target Machine (SI+)
1294//===----------------------------------------------------------------------===//
1295
1296GCNTargetMachine::GCNTargetMachine(const Target &T, const Triple &TT,
1297 StringRef CPU, StringRef FS,
1298 const TargetOptions &Options,
1299 std::optional<Reloc::Model> RM,
1300 std::optional<CodeModel::Model> CM,
1301 CodeGenOptLevel OL, bool JIT)
1302 : AMDGPUTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {
1303 setEnableDefaultMachineVerifier(false);
1304}
1305
1306enum class OOBFlagValue {
1307 Any = 0,
1308 Relaxed = 1,
1309 Strict = 2,
1310};
1311
1312/// Returns the OOB mode encoded by a module flag.
1313/// An absent flag defaults to Any.
1314static OOBFlagValue getOOBFlagValue(const Module &M, StringRef FlagName) {
1315 const auto *Flag =
1316 mdconst::dyn_extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: FlagName));
1317 if (!Flag)
1318 return OOBFlagValue::Any;
1319 return static_cast<OOBFlagValue>(Flag->getZExtValue());
1320}
1321
1322/// Returns the xnack/sramecc setting encoded by a module flag.
1323/// Module flag values: 0 = disabled, 1 = enabled.
1324/// An absent flag defaults to Any.
1325AMDGPU::TargetIDSetting
1326GCNTargetMachine::getTargetIDSettingFromModuleFlag(const Module &M,
1327 StringRef FlagName) {
1328 using AMDGPU::TargetIDSetting;
1329
1330 if (XnackSetting.getNumOccurrences() > 0 && FlagName == "amdgpu.xnack")
1331 return XnackSetting ? TargetIDSetting::On : TargetIDSetting::Off;
1332 if (SramEccSetting.getNumOccurrences() > 0 && FlagName == "amdgpu.sramecc")
1333 return SramEccSetting ? TargetIDSetting::On : TargetIDSetting::Off;
1334
1335 const auto *Flag =
1336 mdconst::dyn_extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: FlagName));
1337 if (!Flag)
1338 return TargetIDSetting::Any;
1339 return Flag->getZExtValue() == 0 ? TargetIDSetting::Off : TargetIDSetting::On;
1340}
1341
1342const TargetSubtargetInfo *
1343GCNTargetMachine::getSubtargetImpl(const Function &F) const {
1344 StringRef GPU = getGPUName(F);
1345 StringRef FS = getFeatureString(F);
1346
1347 const Module &M = *F.getParent();
1348 OOBFlagValue BufOOB = getOOBFlagValue(M, FlagName: AMDGPUOOBMode::BufferFlag);
1349 OOBFlagValue TBufOOB = getOOBFlagValue(M, FlagName: AMDGPUOOBMode::TBufferFlag);
1350 bool BufRelaxed = BufOOB == OOBFlagValue::Relaxed;
1351 bool TBufRelaxed = TBufOOB == OOBFlagValue::Relaxed;
1352
1353 using AMDGPU::TargetIDSetting;
1354 TargetIDSetting Xnack = getTargetIDSettingFromModuleFlag(M, FlagName: "amdgpu.xnack");
1355 TargetIDSetting SramEcc =
1356 getTargetIDSettingFromModuleFlag(M, FlagName: "amdgpu.sramecc");
1357
1358 SmallString<128> SubtargetKey(GPU);
1359 SubtargetKey.append(RHS: FS);
1360 if (BufRelaxed)
1361 SubtargetKey.append(RHS: ",buf-oob=1");
1362 if (TBufRelaxed)
1363 SubtargetKey.append(RHS: ",tbuf-oob=1");
1364 if (Xnack != TargetIDSetting::Any) {
1365 SubtargetKey.append(RHS: ",xnack=");
1366 SubtargetKey.push_back(Elt: Xnack == TargetIDSetting::On ? '1' : '0');
1367 }
1368 if (SramEcc != TargetIDSetting::Any) {
1369 SubtargetKey.append(RHS: ",sramecc=");
1370 SubtargetKey.push_back(Elt: Xnack == TargetIDSetting::On ? '1' : '0');
1371 }
1372
1373 auto &I = SubtargetMap[SubtargetKey];
1374 if (!I) {
1375 AMDGPU::GPUKind Kind = AMDGPU::parseArchAMDGCN(CPU: GPU);
1376 Triple::SubArchType GPUSubArch = AMDGPU::getSubArch(AK: Kind);
1377
1378 // Enforce the subtarget is covered by the subarch. Tolerate no subarch for
1379 // legacy compatibility.
1380 const Triple &TT = M.getTargetTriple();
1381 if (GPUSubArch != TT.getSubArch() && Kind != AMDGPU::GK_NONE) {
1382 // Check if this is a generic subarch which has subtargets. Ignore
1383 // unknown subtargets with a known subarch, since for whatever reason
1384 // the convention is to just print a warning and ignore unrecognized
1385 // subtargets.
1386 bool IsLegacyEmptySubArch = TT.getSubArch() == Triple::NoSubArch;
1387 if (!IsLegacyEmptySubArch &&
1388 AMDGPU::getMajorSubArch(SubArch: GPUSubArch) != TT.getSubArch()) {
1389 F.getContext().emitError(ErrorStr: "invalid subtarget '" + Twine(GPU) +
1390 "' for subarch " + TT.getArchName());
1391 }
1392 }
1393
1394 I = std::make_unique<GCNSubtarget>(args: TargetTriple, args&: GPU, args&: FS, args: *this, args&: BufRelaxed,
1395 args&: TBufRelaxed, args&: Xnack, args&: SramEcc);
1396 }
1397
1398 I->setScalarizeGlobalBehavior(ScalarizeGlobal);
1399
1400 return I.get();
1401}
1402
1403TargetTransformInfo
1404GCNTargetMachine::getTargetTransformInfo(const Function &F) const {
1405 return TargetTransformInfo(std::make_unique<GCNTTIImpl>(args: this, args: F));
1406}
1407
1408Error GCNTargetMachine::buildCodeGenPipeline(
1409 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
1410 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
1411 const CGPassBuilderOption &Opts, MCContext &Ctx,
1412 PassInstrumentationCallbacks *PIC) {
1413 AMDGPUCodeGenPassBuilder CGPB(*this, Opts, PIC);
1414 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
1415}
1416
1417ScheduleDAGInstrs *
1418GCNTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
1419 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
1420 if (ST.enableSIScheduler())
1421 return createSIMachineScheduler(C);
1422
1423 StringRef SchedStrategy = AMDGPU::getSchedStrategy(F: C->MF->getFunction());
1424
1425 if (SchedStrategy == "max-ilp")
1426 return createGCNMaxILPMachineScheduler(C);
1427
1428 if (SchedStrategy == "max-memory-clause")
1429 return createGCNMaxMemoryClauseMachineScheduler(C);
1430
1431 if (SchedStrategy == "iterative-ilp")
1432 return createIterativeILPMachineScheduler(C);
1433
1434 if (SchedStrategy == "iterative-minreg")
1435 return createMinRegScheduler(C);
1436
1437 if (SchedStrategy == "iterative-maxocc")
1438 return createIterativeGCNMaxOccupancyMachineScheduler(C);
1439
1440 if (SchedStrategy == "coexec") {
1441 diagnoseUnsupportedCoExecSchedulerSelection(F: C->MF->getFunction(), ST);
1442 return createGCNCoExecMachineScheduler(C);
1443 }
1444
1445 return createGCNMaxOccupancyMachineScheduler(C);
1446}
1447
1448ScheduleDAGInstrs *
1449GCNTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
1450 if (useNoopPostScheduler(F: C->MF->getFunction()))
1451 return createGCNNoopPostMachineScheduler(C);
1452
1453 ScheduleDAGMI *DAG =
1454 new GCNPostScheduleDAGMILive(C, std::make_unique<PostGenericScheduler>(args&: C),
1455 /*RemoveKillFlags=*/true);
1456 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
1457 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
1458 if (ST.shouldClusterStores())
1459 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
1460 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::PostRA));
1461 if ((EnableVOPD.getNumOccurrences() ||
1462 getOptLevel() >= CodeGenOptLevel::Less) &&
1463 EnableVOPD)
1464 DAG->addMutation(Mutation: createVOPDPairingMutation());
1465 DAG->addMutation(Mutation: createAMDGPUExportClusteringDAGMutation());
1466 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
1467 DAG->addMutation(Mutation: createAMDGPUHazardLatencyDAGMutation(MF: C->MF));
1468 return DAG;
1469}
1470//===----------------------------------------------------------------------===//
1471// AMDGPU Legacy Pass Setup
1472//===----------------------------------------------------------------------===//
1473
1474std::unique_ptr<CSEConfigBase> llvm::AMDGPUPassConfig::getCSEConfig() const {
1475 return getStandardCSEConfigForOpt(Level: TM->getOptLevel());
1476}
1477
1478namespace {
1479
1480class GCNPassConfig final : public AMDGPUPassConfig {
1481public:
1482 GCNPassConfig(TargetMachine &TM, PassManagerBase &PM)
1483 : AMDGPUPassConfig(TM, PM) {
1484 substitutePass(StandardID: &PostRASchedulerID, TargetID: &PostMachineSchedulerID);
1485 }
1486
1487 GCNTargetMachine &getGCNTargetMachine() const {
1488 return getTM<GCNTargetMachine>();
1489 }
1490
1491 bool addPreISel() override;
1492 void addMachineSSAOptimization() override;
1493 bool addILPOpts() override;
1494 bool addInstSelector() override;
1495 bool addIRTranslator() override;
1496 void addPreLegalizeMachineIR() override;
1497 bool addLegalizeMachineIR() override;
1498 void addPreRegBankSelect() override;
1499 bool addRegBankSelect() override;
1500 void addPreGlobalInstructionSelect() override;
1501 bool addGlobalInstructionSelect() override;
1502 void addPreRegAlloc() override;
1503 void addFastRegAlloc() override;
1504 void addOptimizedRegAlloc() override;
1505
1506 FunctionPass *createSGPRAllocPass(bool Optimized);
1507 FunctionPass *createVGPRAllocPass(bool Optimized);
1508 FunctionPass *createWWMRegAllocPass(bool Optimized);
1509 FunctionPass *createRegAllocPass(bool Optimized) override;
1510
1511 bool addRegAssignAndRewriteFast() override;
1512 bool addRegAssignAndRewriteOptimized() override;
1513
1514 bool addPreRewrite() override;
1515 void addPostRegAlloc() override;
1516 void addPreSched2() override;
1517 void addPreEmitPass() override;
1518 void addPostBBSections() override;
1519};
1520
1521} // end anonymous namespace
1522
1523AMDGPUPassConfig::AMDGPUPassConfig(TargetMachine &TM, PassManagerBase &PM)
1524 : TargetPassConfig(TM, PM) {
1525 // Exceptions and StackMaps are not supported, so these passes will never do
1526 // anything.
1527 disablePass(PassID: &StackMapLivenessID);
1528 disablePass(PassID: &FuncletLayoutID);
1529 // Garbage collection is not supported.
1530 disablePass(PassID: &GCLoweringID);
1531 disablePass(PassID: &ShadowStackGCLoweringID);
1532}
1533
1534void AMDGPUPassConfig::addEarlyCSEOrGVNPass() {
1535 if (getOptLevel() == CodeGenOptLevel::Aggressive)
1536 addPass(P: createGVNPass());
1537 else
1538 addPass(P: createEarlyCSEPass());
1539}
1540
1541void AMDGPUPassConfig::addStraightLineScalarOptimizationPasses() {
1542 if (isPassEnabled(Opt: EnableLoopPrefetch, Level: CodeGenOptLevel::Aggressive))
1543 addPass(P: createLoopDataPrefetchPass());
1544 addPass(P: createSeparateConstOffsetFromGEPPass());
1545 // ReassociateGEPs exposes more opportunities for SLSR. See
1546 // the example in reassociate-geps-and-slsr.ll.
1547 addPass(P: createStraightLineStrengthReducePass());
1548 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or
1549 // EarlyCSE can reuse.
1550 addEarlyCSEOrGVNPass();
1551 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
1552 addPass(P: createNaryReassociatePass());
1553 // NaryReassociate on GEPs creates redundant common expressions, so run
1554 // EarlyCSE after it.
1555 addPass(P: createEarlyCSEPass());
1556}
1557
1558void AMDGPUPassConfig::addIRPasses() {
1559 const AMDGPUTargetMachine &TM = getAMDGPUTargetMachine();
1560
1561 if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN())
1562 addPass(P: createAMDGPURemoveIncompatibleFunctionsPass(&TM));
1563
1564 // There is no reason to run these.
1565 disablePass(PassID: &StackMapLivenessID);
1566 disablePass(PassID: &FuncletLayoutID);
1567 disablePass(PassID: &PatchableFunctionID);
1568
1569 if (TM.getTargetTriple().isAMDGCN())
1570 addPass(P: createAMDGPUPrintfRuntimeBinding());
1571
1572 if (LowerCtorDtor)
1573 addPass(P: createAMDGPUCtorDtorLoweringLegacyPass());
1574
1575 if (TM.getTargetTriple().isAMDGCN() &&
1576 isPassEnabled(Opt: EnableImageIntrinsicOptimizer))
1577 addPass(P: createAMDGPUImageIntrinsicOptimizerPass(&TM));
1578
1579 if (EnableUniformIntrinsicCombine)
1580 addPass(P: createAMDGPUUniformIntrinsicCombineLegacyPass());
1581
1582 // This can be disabled by passing ::Disable here or on the command line
1583 // with --expand-variadics-override=disable.
1584 addPass(P: createExpandVariadicsPass(ExpandVariadicsMode::Lowering));
1585
1586 // Function calls are not supported, so make sure we inline everything.
1587 addPass(P: createAMDGPUAlwaysInlinePass());
1588 addPass(P: createAlwaysInlinerLegacyPass());
1589
1590 // Handle uses of OpenCL image2d_t, image3d_t and sampler_t arguments.
1591 if (TM.getTargetTriple().getArch() == Triple::r600)
1592 addPass(P: createR600OpenCLImageTypeLoweringPass());
1593
1594 // Make enqueued block runtime handles externally visible.
1595 addPass(P: createAMDGPUExportKernelRuntimeHandlesLegacyPass());
1596
1597 // Lower special LDS accesses.
1598 if (EnableLowerExecSync)
1599 addPass(P: createAMDGPULowerExecSyncLegacyPass());
1600
1601 // Lower LDS accesses to global memory pass if address sanitizer is enabled.
1602 if (EnableSwLowerLDS)
1603 addPass(P: createAMDGPUSwLowerLDSLegacyPass());
1604
1605 // Runs before PromoteAlloca so the latter can account for function uses
1606 if (EnableLowerModuleLDS) {
1607 addPass(P: createAMDGPULowerModuleLDSLegacyPass(TM: &TM));
1608 }
1609
1610 // Run atomic optimizer before Atomic Expand
1611 if ((TM.getTargetTriple().isAMDGCN()) &&
1612 (TM.getOptLevel() >= CodeGenOptLevel::Less) &&
1613 (AMDGPUAtomicOptimizerStrategy != ScanOptions::None)) {
1614 addPass(P: createAMDGPUAtomicOptimizerPass(ScanStrategy: AMDGPUAtomicOptimizerStrategy));
1615 }
1616
1617 addPass(P: createAtomicExpandLegacyPass());
1618
1619 if (TM.getOptLevel() > CodeGenOptLevel::None) {
1620 addPass(P: createAMDGPUPromoteAlloca());
1621
1622 if (isPassEnabled(Opt: EnableScalarIRPasses))
1623 addStraightLineScalarOptimizationPasses();
1624
1625 if (EnableAMDGPUAliasAnalysis) {
1626 addPass(P: createAMDGPUAAWrapperPass());
1627 addPass(P: createExternalAAWrapperPass(Callback: [](Pass &P, Function &,
1628 AAResults &AAR) {
1629 if (auto *WrapperPass = P.getAnalysisIfAvailable<AMDGPUAAWrapperPass>())
1630 AAR.addAAResult(AAResult&: WrapperPass->getResult());
1631 }));
1632 }
1633
1634 if (TM.getTargetTriple().isAMDGCN()) {
1635 // TODO: May want to move later or split into an early and late one.
1636 addPass(P: createAMDGPUCodeGenPreparePass());
1637 }
1638
1639 // Try to hoist loop invariant parts of divisions AMDGPUCodeGenPrepare may
1640 // have expanded.
1641 if (TM.getOptLevel() > CodeGenOptLevel::Less)
1642 addPass(P: createLICMPass());
1643 }
1644
1645 TargetPassConfig::addIRPasses();
1646
1647 // EarlyCSE is not always strong enough to clean up what LSR produces. For
1648 // example, GVN can combine
1649 //
1650 // %0 = add %a, %b
1651 // %1 = add %b, %a
1652 //
1653 // and
1654 //
1655 // %0 = shl nsw %a, 2
1656 // %1 = shl %a, 2
1657 //
1658 // but EarlyCSE can do neither of them.
1659 if (isPassEnabled(Opt: EnableScalarIRPasses))
1660 addEarlyCSEOrGVNPass();
1661}
1662
1663void AMDGPUPassConfig::addCodeGenPrepare() {
1664 if (TM->getTargetTriple().isAMDGCN() &&
1665 TM->getOptLevel() > CodeGenOptLevel::None)
1666 addPass(P: createAMDGPUPreloadKernelArgumentsLegacyPass(TM));
1667
1668 if (TM->getTargetTriple().isAMDGCN() && EnableLowerKernelArguments)
1669 addPass(P: createAMDGPULowerKernelArgumentsPass());
1670
1671 TargetPassConfig::addCodeGenPrepare();
1672
1673 if (isPassEnabled(Opt: EnableLoadStoreVectorizer))
1674 addPass(P: createLoadStoreVectorizerPass());
1675
1676 if (TM->getTargetTriple().isAMDGCN()) {
1677 // This lowering has been placed after codegenprepare to take advantage of
1678 // address mode matching (which is why it isn't put with the LDS lowerings).
1679 // It could be placed anywhere before uniformity annotations (an analysis
1680 // that it changes by splitting up fat pointers into their components)
1681 // but has been put before switch lowering and CFG flattening so that those
1682 // passes can run on the more optimized control flow this pass creates in
1683 // many cases.
1684 addPass(P: createAMDGPULowerBufferFatPointersPass());
1685 addPass(P: createAMDGPULowerIntrinsicsLegacyPass());
1686 }
1687
1688 // LowerSwitch pass may introduce unreachable blocks that can
1689 // cause unexpected behavior for subsequent passes. Placing it
1690 // here seems better that these blocks would get cleaned up by
1691 // UnreachableBlockElim inserted next in the pass flow.
1692 addPass(P: createLowerSwitchPass());
1693}
1694
1695bool AMDGPUPassConfig::addPreISel() {
1696 if (TM->getOptLevel() > CodeGenOptLevel::None)
1697 addPass(P: createFlattenCFGPass());
1698 return false;
1699}
1700
1701bool AMDGPUPassConfig::addInstSelector() {
1702 addPass(P: createAMDGPUISelDag(TM&: getAMDGPUTargetMachine(), OptLevel: getOptLevel()));
1703 return false;
1704}
1705
1706bool AMDGPUPassConfig::addGCPasses() {
1707 // Do nothing. GC is not supported.
1708 return false;
1709}
1710
1711//===----------------------------------------------------------------------===//
1712// GCN Legacy Pass Setup
1713//===----------------------------------------------------------------------===//
1714
1715bool GCNPassConfig::addPreISel() {
1716 AMDGPUPassConfig::addPreISel();
1717
1718 if (TM->getOptLevel() > CodeGenOptLevel::None) {
1719 addPass(P: createSinkingPass());
1720 addPass(P: createAMDGPULateCodeGenPrepareLegacyPass());
1721 }
1722
1723 // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
1724 // regions formed by them.
1725 addPass(PassID: &AMDGPUUnifyDivergentExitNodesID);
1726 addPass(P: createFixIrreduciblePass());
1727 addPass(P: createUnifyLoopExitsPass());
1728 addPass(P: createStructurizeCFGPass(SkipUniformRegions: false)); // true -> SkipUniformRegions
1729
1730 addPass(P: createAMDGPUAnnotateUniformValuesLegacy());
1731 addPass(P: createSIAnnotateControlFlowLegacyPass());
1732 // TODO: Move this right after structurizeCFG to avoid extra divergence
1733 // analysis. This depends on stopping SIAnnotateControlFlow from making
1734 // control flow modifications.
1735 addPass(P: createAMDGPURewriteUndefForPHILegacyPass());
1736
1737 // SDAG requires LCSSA, GlobalISel does not. Disable LCSSA for -global-isel
1738 // without any of the fallback options.
1739 if (getCGPassBuilderOption().EnableGlobalISelOption !=
1740 cl::boolOrDefault::BOU_TRUE ||
1741 !isGlobalISelAbortEnabled())
1742 addPass(P: createLCSSAPass());
1743
1744 if (TM->getOptLevel() > CodeGenOptLevel::Less)
1745 addPass(PassID: &AMDGPUPerfHintAnalysisLegacyID);
1746
1747 return false;
1748}
1749
1750void GCNPassConfig::addMachineSSAOptimization() {
1751 TargetPassConfig::addMachineSSAOptimization();
1752
1753 // We want to fold operands after PeepholeOptimizer has run (or as part of
1754 // it), because it will eliminate extra copies making it easier to fold the
1755 // real source operand. We want to eliminate dead instructions after, so that
1756 // we see fewer uses of the copies. We then need to clean up the dead
1757 // instructions leftover after the operands are folded as well.
1758 //
1759 // XXX - Can we get away without running DeadMachineInstructionElim again?
1760 addPass(PassID: &SIFoldOperandsLegacyID);
1761 if (EnableDPPCombine)
1762 addPass(PassID: &GCNDPPCombineLegacyID);
1763 addPass(PassID: &SILoadStoreOptimizerLegacyID);
1764 if (isPassEnabled(Opt: EnableSDWAPeephole)) {
1765 addPass(PassID: &SIPeepholeSDWALegacyID);
1766 addPass(PassID: &EarlyMachineLICMID);
1767 addPass(PassID: &MachineCSELegacyID);
1768 addPass(PassID: &SIFoldOperandsLegacyID);
1769 }
1770 addPass(PassID: &DeadMachineInstructionElimID);
1771 addPass(P: createSIShrinkInstructionsLegacyPass());
1772}
1773
1774bool GCNPassConfig::addILPOpts() {
1775 if (EnableEarlyIfConversion)
1776 addPass(PassID: &EarlyIfConverterLegacyID);
1777
1778 TargetPassConfig::addILPOpts();
1779 return false;
1780}
1781
1782bool GCNPassConfig::addInstSelector() {
1783 AMDGPUPassConfig::addInstSelector();
1784 addPass(PassID: &SIFixSGPRCopiesLegacyID);
1785 addPass(P: createSILowerI1CopiesLegacyPass());
1786 return false;
1787}
1788
1789bool GCNPassConfig::addIRTranslator() {
1790 addPass(P: new IRTranslatorLegacy(getOptLevel()));
1791 return false;
1792}
1793
1794void GCNPassConfig::addPreLegalizeMachineIR() {
1795 bool IsOptNone = getOptLevel() == CodeGenOptLevel::None;
1796 addPass(P: createAMDGPUPreLegalizeCombiner(IsOptNone));
1797 addPass(P: new LocalizerLegacy());
1798}
1799
1800bool GCNPassConfig::addLegalizeMachineIR() {
1801 addPass(P: new LegalizerLegacy());
1802 return false;
1803}
1804
1805void GCNPassConfig::addPreRegBankSelect() {
1806 bool IsOptNone = getOptLevel() == CodeGenOptLevel::None;
1807 addPass(P: createAMDGPUPostLegalizeCombiner(IsOptNone));
1808 addPass(P: createAMDGPUGlobalISelDivergenceLoweringPass());
1809}
1810
1811bool GCNPassConfig::addRegBankSelect() {
1812 addPass(P: createAMDGPURegBankSelectPass());
1813 addPass(P: createAMDGPURegBankLegalizePass());
1814 return false;
1815}
1816
1817void GCNPassConfig::addPreGlobalInstructionSelect() {
1818 bool IsOptNone = getOptLevel() == CodeGenOptLevel::None;
1819 addPass(P: createAMDGPURegBankCombiner(IsOptNone));
1820}
1821
1822bool GCNPassConfig::addGlobalInstructionSelect() {
1823 addPass(P: new InstructionSelectLegacy(getOptLevel()));
1824 return false;
1825}
1826
1827void GCNPassConfig::addFastRegAlloc() {
1828 // FIXME: We have to disable the verifier here because of PHIElimination +
1829 // TwoAddressInstructions disabling it.
1830
1831 // This must be run immediately after phi elimination and before
1832 // TwoAddressInstructions, otherwise the processing of the tied operand of
1833 // SI_ELSE will introduce a copy of the tied operand source after the else.
1834 insertPass(TargetPassID: &PHIEliminationID, InsertedPassID: &SILowerControlFlowLegacyID);
1835
1836 insertPass(TargetPassID: &TwoAddressInstructionPassID, InsertedPassID: &SIWholeQuadModeID);
1837
1838 TargetPassConfig::addFastRegAlloc();
1839}
1840
1841void GCNPassConfig::addPreRegAlloc() {
1842 if (getOptLevel() != CodeGenOptLevel::None)
1843 addPass(PassID: &AMDGPUPrepareAGPRAllocLegacyID);
1844 if (getOptLevel() >= CodeGenOptLevel::Default && EnableMachinePipeliner)
1845 addPass(PassID: &MachinePipelinerID);
1846}
1847
1848void GCNPassConfig::addOptimizedRegAlloc() {
1849 if (EnableDCEInRA)
1850 insertPass(TargetPassID: &DetectDeadLanesID, InsertedPassID: &DeadMachineInstructionElimID);
1851
1852 // FIXME: when an instruction has a Killed operand, and the instruction is
1853 // inside a bundle, seems only the BUNDLE instruction appears as the Kills of
1854 // the register in LiveVariables, this would trigger a failure in verifier,
1855 // we should fix it and enable the verifier.
1856 if (OptVGPRLiveRange)
1857 insertPass(TargetPassID: &LiveVariablesID, InsertedPassID: &SIOptimizeVGPRLiveRangeLegacyID);
1858
1859 // This must be run immediately after phi elimination and before
1860 // TwoAddressInstructions, otherwise the processing of the tied operand of
1861 // SI_ELSE will introduce a copy of the tied operand source after the else.
1862 insertPass(TargetPassID: &PHIEliminationID, InsertedPassID: &SILowerControlFlowLegacyID);
1863
1864 if (EnableRewritePartialRegUses)
1865 insertPass(TargetPassID: &RenameIndependentSubregsID, InsertedPassID: &GCNRewritePartialRegUsesID);
1866
1867 if (isPassEnabled(Opt: EnablePreRAOptimizations))
1868 insertPass(TargetPassID: &MachineSchedulerID, InsertedPassID: &GCNPreRAOptimizationsID);
1869
1870 // Allow the scheduler to run before SIWholeQuadMode inserts exec manipulation
1871 // instructions that cause scheduling barriers.
1872 insertPass(TargetPassID: &MachineSchedulerID, InsertedPassID: &SIWholeQuadModeID);
1873
1874 if (OptExecMaskPreRA)
1875 insertPass(TargetPassID: &MachineSchedulerID, InsertedPassID: &SIOptimizeExecMaskingPreRAID);
1876
1877 // This is not an essential optimization and it has a noticeable impact on
1878 // compilation time, so we only enable it from O2.
1879 if (TM->getOptLevel() > CodeGenOptLevel::Less)
1880 insertPass(TargetPassID: &MachineSchedulerID, InsertedPassID: &SIFormMemoryClausesID);
1881
1882 TargetPassConfig::addOptimizedRegAlloc();
1883}
1884
1885bool GCNPassConfig::addPreRewrite() {
1886 if (EnableRegReassign)
1887 addPass(PassID: &GCNNSAReassignID);
1888
1889 addPass(PassID: &AMDGPURewriteAGPRCopyMFMALegacyID);
1890 return true;
1891}
1892
1893FunctionPass *GCNPassConfig::createSGPRAllocPass(bool Optimized) {
1894 // Initialize the global default.
1895 llvm::call_once(flag&: InitializeDefaultSGPRRegisterAllocatorFlag,
1896 F&: initializeDefaultSGPRRegisterAllocatorOnce);
1897
1898 RegisterRegAlloc::FunctionPassCtor Ctor = SGPRRegisterRegAlloc::getDefault();
1899 if (Ctor != useDefaultRegisterAllocator)
1900 return Ctor();
1901
1902 if (Optimized)
1903 return createGreedyRegisterAllocator(F: onlyAllocateSGPRs);
1904
1905 return createFastRegisterAllocator(F: onlyAllocateSGPRs, ClearVirtRegs: false);
1906}
1907
1908FunctionPass *GCNPassConfig::createVGPRAllocPass(bool Optimized) {
1909 // Initialize the global default.
1910 llvm::call_once(flag&: InitializeDefaultVGPRRegisterAllocatorFlag,
1911 F&: initializeDefaultVGPRRegisterAllocatorOnce);
1912
1913 RegisterRegAlloc::FunctionPassCtor Ctor = VGPRRegisterRegAlloc::getDefault();
1914 if (Ctor != useDefaultRegisterAllocator)
1915 return Ctor();
1916
1917 if (Optimized)
1918 return createGreedyVGPRRegisterAllocator();
1919
1920 return createFastVGPRRegisterAllocator();
1921}
1922
1923FunctionPass *GCNPassConfig::createWWMRegAllocPass(bool Optimized) {
1924 // Initialize the global default.
1925 llvm::call_once(flag&: InitializeDefaultWWMRegisterAllocatorFlag,
1926 F&: initializeDefaultWWMRegisterAllocatorOnce);
1927
1928 RegisterRegAlloc::FunctionPassCtor Ctor = WWMRegisterRegAlloc::getDefault();
1929 if (Ctor != useDefaultRegisterAllocator)
1930 return Ctor();
1931
1932 if (Optimized)
1933 return createGreedyWWMRegisterAllocator();
1934
1935 return createFastWWMRegisterAllocator();
1936}
1937
1938FunctionPass *GCNPassConfig::createRegAllocPass(bool Optimized) {
1939 llvm_unreachable("should not be used");
1940}
1941
1942static const char RegAllocOptNotSupportedMessage[] =
1943 "-regalloc not supported with amdgcn. Use -sgpr-regalloc, -wwm-regalloc, "
1944 "and -vgpr-regalloc";
1945
1946bool GCNPassConfig::addRegAssignAndRewriteFast() {
1947 if (!usingDefaultRegAlloc())
1948 reportFatalUsageError(reason: RegAllocOptNotSupportedMessage);
1949
1950 addPass(PassID: &GCNPreRALongBranchRegID);
1951
1952 addPass(P: createSGPRAllocPass(Optimized: false));
1953
1954 // Equivalent of PEI for SGPRs.
1955 addPass(PassID: &SILowerSGPRSpillsLegacyID);
1956
1957 // To Allocate wwm registers used in whole quad mode operations (for shaders).
1958 addPass(PassID: &SIPreAllocateWWMRegsLegacyID);
1959
1960 // For allocating other wwm register operands.
1961 addPass(P: createWWMRegAllocPass(Optimized: false));
1962
1963 addPass(PassID: &SILowerWWMCopiesLegacyID);
1964 addPass(PassID: &AMDGPUReserveWWMRegsLegacyID);
1965
1966 // For allocating per-thread VGPRs.
1967 addPass(P: createVGPRAllocPass(Optimized: false));
1968
1969 return true;
1970}
1971
1972bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
1973 if (!usingDefaultRegAlloc())
1974 reportFatalUsageError(reason: RegAllocOptNotSupportedMessage);
1975
1976 addPass(PassID: &GCNPreRALongBranchRegID);
1977
1978 addPass(P: createSGPRAllocPass(Optimized: true));
1979
1980 // Commit allocated register changes. This is mostly necessary because too
1981 // many things rely on the use lists of the physical registers, such as the
1982 // verifier. This is only necessary with allocators which use LiveIntervals,
1983 // since FastRegAlloc does the replacements itself.
1984 addPass(P: createVirtRegRewriter(ClearVirtRegs: false));
1985
1986 // At this point, the sgpr-regalloc has been done and it is good to have the
1987 // stack slot coloring to try to optimize the SGPR spill stack indices before
1988 // attempting the custom SGPR spill lowering.
1989 addPass(PassID: &StackSlotColoringID);
1990
1991 // Equivalent of PEI for SGPRs.
1992 addPass(PassID: &SILowerSGPRSpillsLegacyID);
1993
1994 // To Allocate wwm registers used in whole quad mode operations (for shaders).
1995 addPass(PassID: &SIPreAllocateWWMRegsLegacyID);
1996
1997 // For allocating other whole wave mode registers.
1998 addPass(P: createWWMRegAllocPass(Optimized: true));
1999 addPass(PassID: &SILowerWWMCopiesLegacyID);
2000 addPass(P: createVirtRegRewriter(ClearVirtRegs: false));
2001 addPass(PassID: &AMDGPUReserveWWMRegsLegacyID);
2002
2003 // For allocating per-thread VGPRs.
2004 addPass(P: createVGPRAllocPass(Optimized: true));
2005
2006 addPreRewrite();
2007 addPass(PassID: &VirtRegRewriterID);
2008
2009 addPass(PassID: &AMDGPUMarkLastScratchLoadID);
2010
2011 return true;
2012}
2013
2014void GCNPassConfig::addPostRegAlloc() {
2015 addPass(PassID: &SIFixVGPRCopiesID);
2016 if (getOptLevel() > CodeGenOptLevel::None)
2017 addPass(PassID: &SIOptimizeExecMaskingLegacyID);
2018 TargetPassConfig::addPostRegAlloc();
2019}
2020
2021void GCNPassConfig::addPreSched2() {
2022 if (TM->getOptLevel() > CodeGenOptLevel::None)
2023 addPass(P: createSIShrinkInstructionsLegacyPass());
2024 addPass(PassID: &SIPostRABundlerLegacyID);
2025}
2026
2027void GCNPassConfig::addPreEmitPass() {
2028 if (isPassEnabled(Opt: EnableVOPD, Level: CodeGenOptLevel::Less))
2029 addPass(PassID: &GCNCreateVOPDID);
2030 addPass(P: createSIMemoryLegalizerPass());
2031 addPass(P: createSIInsertWaitcntsPass());
2032
2033 addPass(P: createSIModeRegisterPass());
2034
2035 if (getOptLevel() > CodeGenOptLevel::None)
2036 addPass(PassID: &SIInsertHardClausesID);
2037
2038 addPass(PassID: &SILateBranchLoweringPassID);
2039 if (isPassEnabled(Opt: EnableSetWavePriority, Level: CodeGenOptLevel::Less))
2040 addPass(P: createAMDGPUSetWavePriorityPass());
2041 if (getOptLevel() > CodeGenOptLevel::None)
2042 addPass(PassID: &SIPreEmitPeepholeID);
2043 // The hazard recognizer that runs as part of the post-ra scheduler does not
2044 // guarantee to be able handle all hazards correctly. This is because if there
2045 // are multiple scheduling regions in a basic block, the regions are scheduled
2046 // bottom up, so when we begin to schedule a region we don't know what
2047 // instructions were emitted directly before it.
2048 //
2049 // Here we add a stand-alone hazard recognizer pass which can handle all
2050 // cases.
2051 addPass(PassID: &PostRAHazardRecognizerID);
2052
2053 addPass(PassID: &AMDGPUWaitSGPRHazardsLegacyID);
2054
2055 addPass(PassID: &AMDGPULowerVGPREncodingLegacyID);
2056
2057 if (isPassEnabled(Opt: EnableInsertDelayAlu, Level: CodeGenOptLevel::Less))
2058 addPass(PassID: &AMDGPUInsertDelayAluID);
2059
2060 addPass(PassID: &BranchRelaxationPassID);
2061}
2062
2063void GCNPassConfig::addPostBBSections() {
2064 // We run this later to avoid passes like livedebugvalues and BBSections
2065 // having to deal with the apparent multi-entry functions we may generate.
2066 addPass(P: createAMDGPUPreloadKernArgPrologLegacyPass());
2067}
2068
2069TargetPassConfig *GCNTargetMachine::createPassConfig(PassManagerBase &PM) {
2070 return new GCNPassConfig(*this, PM);
2071}
2072
2073void GCNTargetMachine::registerMachineRegisterInfoCallback(
2074 MachineFunction &MF) const {
2075 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2076 MF.getRegInfo().addDelegate(delegate: MFI);
2077}
2078
2079MachineFunctionInfo *GCNTargetMachine::createMachineFunctionInfo(
2080 BumpPtrAllocator &Allocator, const Function &F,
2081 const TargetSubtargetInfo *STI) const {
2082 return SIMachineFunctionInfo::create<SIMachineFunctionInfo>(
2083 Allocator, F, STI: static_cast<const GCNSubtarget *>(STI));
2084}
2085
2086yaml::MachineFunctionInfo *GCNTargetMachine::createDefaultFuncInfoYAML() const {
2087 return new yaml::SIMachineFunctionInfo();
2088}
2089
2090yaml::MachineFunctionInfo *
2091GCNTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
2092 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2093 return new yaml::SIMachineFunctionInfo(
2094 *MFI, *MF.getSubtarget<GCNSubtarget>().getRegisterInfo(), MF);
2095}
2096
2097bool GCNTargetMachine::parseMachineFunctionInfo(
2098 const yaml::MachineFunctionInfo &MFI_, PerFunctionMIParsingState &PFS,
2099 SMDiagnostic &Error, SMRange &SourceRange) const {
2100 const yaml::SIMachineFunctionInfo &YamlMFI =
2101 static_cast<const yaml::SIMachineFunctionInfo &>(MFI_);
2102 MachineFunction &MF = PFS.MF;
2103 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2104 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2105
2106 if (MFI->initializeBaseYamlFields(YamlMFI, MF, PFS, Error, SourceRange))
2107 return true;
2108
2109 if (MFI->Occupancy == 0) {
2110 // Fixup the subtarget dependent default value.
2111 MFI->Occupancy = ST.getOccupancyWithWorkGroupSizes(MF).second;
2112 }
2113
2114 auto parseRegister = [&](const yaml::StringValue &RegName, Register &RegVal) {
2115 Register TempReg;
2116 if (parseNamedRegisterReference(PFS, Reg&: TempReg, Src: RegName.Value, Error)) {
2117 SourceRange = RegName.SourceRange;
2118 return true;
2119 }
2120 RegVal = TempReg;
2121
2122 return false;
2123 };
2124
2125 auto parseOptionalRegister = [&](const yaml::StringValue &RegName,
2126 Register &RegVal) {
2127 return !RegName.Value.empty() && parseRegister(RegName, RegVal);
2128 };
2129
2130 if (parseOptionalRegister(YamlMFI.VGPRForAGPRCopy, MFI->VGPRForAGPRCopy))
2131 return true;
2132
2133 if (parseOptionalRegister(YamlMFI.SGPRForEXECCopy, MFI->SGPRForEXECCopy))
2134 return true;
2135
2136 if (parseOptionalRegister(YamlMFI.LongBranchReservedReg,
2137 MFI->LongBranchReservedReg))
2138 return true;
2139
2140 auto diagnoseRegisterClass = [&](const yaml::StringValue &RegName) {
2141 // Create a diagnostic for a the register string literal.
2142 const MemoryBuffer &Buffer =
2143 *PFS.SM->getMemoryBuffer(i: PFS.SM->getMainFileID());
2144 Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1,
2145 RegName.Value.size(), SourceMgr::DK_Error,
2146 "incorrect register class for field", RegName.Value,
2147 {}, {});
2148 SourceRange = RegName.SourceRange;
2149 return true;
2150 };
2151
2152 if (parseRegister(YamlMFI.ScratchRSrcReg, MFI->ScratchRSrcReg) ||
2153 parseRegister(YamlMFI.FrameOffsetReg, MFI->FrameOffsetReg) ||
2154 parseRegister(YamlMFI.StackPtrOffsetReg, MFI->StackPtrOffsetReg))
2155 return true;
2156
2157 if (MFI->ScratchRSrcReg != AMDGPU::PRIVATE_RSRC_REG &&
2158 !AMDGPU::SGPR_128RegClass.contains(Reg: MFI->ScratchRSrcReg)) {
2159 return diagnoseRegisterClass(YamlMFI.ScratchRSrcReg);
2160 }
2161
2162 if (MFI->FrameOffsetReg != AMDGPU::FP_REG &&
2163 !AMDGPU::SGPR_32RegClass.contains(Reg: MFI->FrameOffsetReg)) {
2164 return diagnoseRegisterClass(YamlMFI.FrameOffsetReg);
2165 }
2166
2167 if (MFI->StackPtrOffsetReg != AMDGPU::SP_REG &&
2168 !AMDGPU::SGPR_32RegClass.contains(Reg: MFI->StackPtrOffsetReg)) {
2169 return diagnoseRegisterClass(YamlMFI.StackPtrOffsetReg);
2170 }
2171
2172 for (const auto &YamlReg : YamlMFI.WWMReservedRegs) {
2173 Register ParsedReg;
2174 if (parseRegister(YamlReg, ParsedReg))
2175 return true;
2176
2177 MFI->reserveWWMRegister(Reg: ParsedReg);
2178 }
2179
2180 for (const auto &[_, Info] : PFS.VRegInfosNamed) {
2181 MFI->setFlag(Reg: Info->VReg, Flag: Info->Flags);
2182 }
2183 for (const auto &[_, Info] : PFS.VRegInfos) {
2184 MFI->setFlag(Reg: Info->VReg, Flag: Info->Flags);
2185 }
2186
2187 for (const auto &YamlRegStr : YamlMFI.SpillPhysVGPRS) {
2188 Register ParsedReg;
2189 if (parseRegister(YamlRegStr, ParsedReg))
2190 return true;
2191 MFI->SpillPhysVGPRs.push_back(Elt: ParsedReg);
2192 }
2193
2194 auto parseAndCheckArgument = [&](const std::optional<yaml::SIArgument> &A,
2195 const TargetRegisterClass &RC,
2196 ArgDescriptor &Arg, unsigned UserSGPRs,
2197 unsigned SystemSGPRs) {
2198 // Skip parsing if it's not present.
2199 if (!A)
2200 return false;
2201
2202 if (A->IsRegister) {
2203 Register Reg;
2204 if (parseNamedRegisterReference(PFS, Reg, Src: A->RegisterName.Value, Error)) {
2205 SourceRange = A->RegisterName.SourceRange;
2206 return true;
2207 }
2208 if (!RC.contains(Reg))
2209 return diagnoseRegisterClass(A->RegisterName);
2210 Arg = ArgDescriptor::createRegister(Reg);
2211 } else
2212 Arg = ArgDescriptor::createStack(Offset: A->StackOffset);
2213 // Check and apply the optional mask.
2214 if (A->Mask)
2215 Arg = ArgDescriptor::createArg(Arg, Mask: *A->Mask);
2216
2217 MFI->NumUserSGPRs += UserSGPRs;
2218 MFI->NumSystemSGPRs += SystemSGPRs;
2219 return false;
2220 };
2221
2222 if (YamlMFI.ArgInfo &&
2223 (parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentBuffer,
2224 AMDGPU::SGPR_128RegClass,
2225 MFI->ArgInfo.PrivateSegmentBuffer, 4, 0) ||
2226 parseAndCheckArgument(YamlMFI.ArgInfo->DispatchPtr,
2227 AMDGPU::SReg_64RegClass, MFI->ArgInfo.DispatchPtr,
2228 2, 0) ||
2229 parseAndCheckArgument(YamlMFI.ArgInfo->QueuePtr, AMDGPU::SReg_64RegClass,
2230 MFI->ArgInfo.QueuePtr, 2, 0) ||
2231 parseAndCheckArgument(YamlMFI.ArgInfo->KernargSegmentPtr,
2232 AMDGPU::SReg_64RegClass,
2233 MFI->ArgInfo.KernargSegmentPtr, 2, 0) ||
2234 parseAndCheckArgument(YamlMFI.ArgInfo->DispatchID,
2235 AMDGPU::SReg_64RegClass, MFI->ArgInfo.DispatchID,
2236 2, 0) ||
2237 parseAndCheckArgument(YamlMFI.ArgInfo->FlatScratchInit,
2238 AMDGPU::SReg_64RegClass,
2239 MFI->ArgInfo.FlatScratchInit, 2, 0) ||
2240 parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentSize,
2241 AMDGPU::SGPR_32RegClass,
2242 MFI->ArgInfo.PrivateSegmentSize, 0, 0) ||
2243 parseAndCheckArgument(YamlMFI.ArgInfo->LDSKernelId,
2244 AMDGPU::SGPR_32RegClass,
2245 MFI->ArgInfo.LDSKernelId, 0, 1) ||
2246 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDX,
2247 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDX,
2248 0, 1) ||
2249 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDY,
2250 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDY,
2251 0, 1) ||
2252 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDZ,
2253 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDZ,
2254 0, 1) ||
2255 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupInfo,
2256 AMDGPU::SGPR_32RegClass,
2257 MFI->ArgInfo.WorkGroupInfo, 0, 1) ||
2258 parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentWaveByteOffset,
2259 AMDGPU::SGPR_32RegClass,
2260 MFI->ArgInfo.PrivateSegmentWaveByteOffset, 0, 1) ||
2261 parseAndCheckArgument(YamlMFI.ArgInfo->ImplicitArgPtr,
2262 AMDGPU::SReg_64RegClass,
2263 MFI->ArgInfo.ImplicitArgPtr, 0, 0) ||
2264 parseAndCheckArgument(YamlMFI.ArgInfo->ImplicitBufferPtr,
2265 AMDGPU::SReg_64RegClass,
2266 MFI->ArgInfo.ImplicitBufferPtr, 2, 0) ||
2267 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDX,
2268 AMDGPU::VGPR_32RegClass,
2269 MFI->ArgInfo.WorkItemIDX, 0, 0) ||
2270 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDY,
2271 AMDGPU::VGPR_32RegClass,
2272 MFI->ArgInfo.WorkItemIDY, 0, 0) ||
2273 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDZ,
2274 AMDGPU::VGPR_32RegClass,
2275 MFI->ArgInfo.WorkItemIDZ, 0, 0)))
2276 return true;
2277
2278 // Parse FirstKernArgPreloadReg separately, since it's a Register,
2279 // not ArgDescriptor.
2280 if (YamlMFI.ArgInfo && YamlMFI.ArgInfo->FirstKernArgPreloadReg) {
2281 const yaml::SIArgument &A = *YamlMFI.ArgInfo->FirstKernArgPreloadReg;
2282
2283 if (!A.IsRegister) {
2284 // For stack arguments, we don't have RegisterName.SourceRange,
2285 // but we should have some location info from the YAML parser
2286 const MemoryBuffer &Buffer =
2287 *PFS.SM->getMemoryBuffer(i: PFS.SM->getMainFileID());
2288 // Create a minimal valid source range
2289 SMLoc Loc = SMLoc::getFromPointer(Ptr: Buffer.getBufferStart());
2290 SMRange Range(Loc, Loc);
2291
2292 Error = SMDiagnostic(
2293 *PFS.SM, Loc, Buffer.getBufferIdentifier(), 1, 0, SourceMgr::DK_Error,
2294 "firstKernArgPreloadReg must be a register, not a stack location", "",
2295 {}, {});
2296
2297 SourceRange = Range;
2298 return true;
2299 }
2300
2301 Register Reg;
2302 if (parseNamedRegisterReference(PFS, Reg, Src: A.RegisterName.Value, Error)) {
2303 SourceRange = A.RegisterName.SourceRange;
2304 return true;
2305 }
2306
2307 if (!AMDGPU::SGPR_32RegClass.contains(Reg))
2308 return diagnoseRegisterClass(A.RegisterName);
2309
2310 MFI->ArgInfo.FirstKernArgPreloadReg = Reg;
2311 MFI->NumUserSGPRs += YamlMFI.NumKernargPreloadSGPRs;
2312 }
2313
2314 if (ST.hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode)) {
2315 MFI->Mode.IEEE = YamlMFI.Mode.IEEE;
2316 MFI->Mode.DX10Clamp = YamlMFI.Mode.DX10Clamp;
2317 }
2318
2319 // FIXME: Move proper support for denormal-fp-math into base MachineFunction
2320 MFI->Mode.FP32Denormals.Input = YamlMFI.Mode.FP32InputDenormals
2321 ? DenormalMode::IEEE
2322 : DenormalMode::PreserveSign;
2323 MFI->Mode.FP32Denormals.Output = YamlMFI.Mode.FP32OutputDenormals
2324 ? DenormalMode::IEEE
2325 : DenormalMode::PreserveSign;
2326
2327 MFI->Mode.FP64FP16Denormals.Input = YamlMFI.Mode.FP64FP16InputDenormals
2328 ? DenormalMode::IEEE
2329 : DenormalMode::PreserveSign;
2330 MFI->Mode.FP64FP16Denormals.Output = YamlMFI.Mode.FP64FP16OutputDenormals
2331 ? DenormalMode::IEEE
2332 : DenormalMode::PreserveSign;
2333
2334 if (YamlMFI.HasInitWholeWave)
2335 MFI->setInitWholeWave();
2336
2337 return false;
2338}
2339
2340//===----------------------------------------------------------------------===//
2341// AMDGPU CodeGen Pass Builder interface.
2342//===----------------------------------------------------------------------===//
2343
2344AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
2345 GCNTargetMachine &TM, const CGPassBuilderOption &Opts,
2346 PassInstrumentationCallbacks *PIC)
2347 : CodeGenPassBuilder(TM, Opts, PIC) {
2348 Opt.MISchedPostRA = true;
2349 Opt.RequiresCodeGenSCCOrder = true;
2350 // Exceptions and StackMaps are not supported, so these passes will never do
2351 // anything.
2352 // Garbage collection is not supported.
2353 disablePass<StackMapLivenessPass, FuncletLayoutPass, PatchableFunctionPass,
2354 ShadowStackGCLoweringPass, GCLoweringPass>();
2355}
2356
2357void AMDGPUCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
2358 if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN()) {
2359 flushFPMsToMPM(PMW);
2360 addModulePass(Pass: AMDGPURemoveIncompatibleFunctionsPass(TM), PMW);
2361 }
2362
2363 flushFPMsToMPM(PMW);
2364
2365 if (TM.getTargetTriple().isAMDGCN())
2366 addModulePass(Pass: AMDGPUPrintfRuntimeBindingPass(), PMW);
2367
2368 if (LowerCtorDtor)
2369 addModulePass(Pass: AMDGPUCtorDtorLoweringPass(), PMW);
2370
2371 if (isPassEnabled(Opt: EnableImageIntrinsicOptimizer))
2372 addFunctionPass(Pass: AMDGPUImageIntrinsicOptimizerPass(TM), PMW);
2373
2374 if (EnableUniformIntrinsicCombine)
2375 addFunctionPass(Pass: AMDGPUUniformIntrinsicCombinePass(), PMW);
2376 // This can be disabled by passing ::Disable here or on the command line
2377 // with --expand-variadics-override=disable.
2378 flushFPMsToMPM(PMW);
2379 addModulePass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Lowering), PMW);
2380
2381 addModulePass(Pass: AMDGPUAlwaysInlinePass(), PMW);
2382 addModulePass(Pass: AlwaysInlinerPass(), PMW);
2383
2384 addModulePass(Pass: AMDGPUExportKernelRuntimeHandlesPass(), PMW);
2385
2386 if (EnableLowerExecSync)
2387 addModulePass(Pass: AMDGPULowerExecSyncPass(), PMW);
2388
2389 if (EnableSwLowerLDS)
2390 addModulePass(Pass: AMDGPUSwLowerLDSPass(), PMW);
2391
2392 // Runs before PromoteAlloca so the latter can account for function uses
2393 if (EnableLowerModuleLDS)
2394 addModulePass(Pass: AMDGPULowerModuleLDSPass(getTM()), PMW);
2395
2396 // Run atomic optimizer before Atomic Expand
2397 if (TM.getOptLevel() >= CodeGenOptLevel::Less &&
2398 (AMDGPUAtomicOptimizerStrategy != ScanOptions::None))
2399 addFunctionPass(
2400 Pass: AMDGPUAtomicOptimizerPass(TM, AMDGPUAtomicOptimizerStrategy), PMW);
2401
2402 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
2403
2404 if (TM.getOptLevel() > CodeGenOptLevel::None) {
2405 addFunctionPass(Pass: AMDGPUPromoteAllocaPass(TM), PMW);
2406 if (isPassEnabled(Opt: EnableScalarIRPasses))
2407 addStraightLineScalarOptimizationPasses(PMW);
2408
2409 // TODO: Handle EnableAMDGPUAliasAnalysis
2410
2411 // TODO: May want to move later or split into an early and late one.
2412 addFunctionPass(Pass: AMDGPUCodeGenPreparePass(TM), PMW);
2413
2414 // Try to hoist loop invariant parts of divisions AMDGPUCodeGenPrepare may
2415 // have expanded.
2416 if (TM.getOptLevel() > CodeGenOptLevel::Less) {
2417 addFunctionPass(Pass: createFunctionToLoopPassAdaptor(Pass: LICMPass(LICMOptions()),
2418 /*UseMemorySSA=*/true),
2419 PMW);
2420 }
2421 }
2422
2423 Base::addIRPasses(PMW);
2424
2425 // EarlyCSE is not always strong enough to clean up what LSR produces. For
2426 // example, GVN can combine
2427 //
2428 // %0 = add %a, %b
2429 // %1 = add %b, %a
2430 //
2431 // and
2432 //
2433 // %0 = shl nsw %a, 2
2434 // %1 = shl %a, 2
2435 //
2436 // but EarlyCSE can do neither of them.
2437 if (isPassEnabled(Opt: EnableScalarIRPasses))
2438 addEarlyCSEOrGVNPass(PMW);
2439}
2440
2441void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(PassManagerWrapper &PMW) {
2442 if (TM.getOptLevel() > CodeGenOptLevel::None) {
2443 flushFPMsToMPM(PMW);
2444 addModulePass(Pass: AMDGPUPreloadKernelArgumentsPass(TM), PMW);
2445 }
2446
2447 if (EnableLowerKernelArguments)
2448 addFunctionPass(Pass: AMDGPULowerKernelArgumentsPass(TM), PMW);
2449
2450 Base::addCodeGenPrepare(PMW);
2451
2452 if (isPassEnabled(Opt: EnableLoadStoreVectorizer))
2453 addFunctionPass(Pass: LoadStoreVectorizerPass(), PMW);
2454
2455 // This lowering has been placed after codegenprepare to take advantage of
2456 // address mode matching (which is why it isn't put with the LDS lowerings).
2457 // It could be placed anywhere before uniformity annotations (an analysis
2458 // that it changes by splitting up fat pointers into their components)
2459 // but has been put before switch lowering and CFG flattening so that those
2460 // passes can run on the more optimized control flow this pass creates in
2461 // many cases.
2462 flushFPMsToMPM(PMW);
2463 addModulePass(Pass: AMDGPULowerBufferFatPointersPass(TM), PMW);
2464 flushFPMsToMPM(PMW);
2465 requireCGSCCOrder(PMW);
2466
2467 addModulePass(Pass: AMDGPULowerIntrinsicsPass(getTM()), PMW);
2468
2469 // LowerSwitch pass may introduce unreachable blocks that can cause unexpected
2470 // behavior for subsequent passes. Placing it here seems better that these
2471 // blocks would get cleaned up by UnreachableBlockElim inserted next in the
2472 // pass flow.
2473 addFunctionPass(Pass: LowerSwitchPass(), PMW);
2474}
2475
2476void AMDGPUCodeGenPassBuilder::addPreISel(PassManagerWrapper &PMW) {
2477
2478 if (TM.getOptLevel() > CodeGenOptLevel::None) {
2479 addFunctionPass(Pass: FlattenCFGPass(), PMW);
2480 addFunctionPass(Pass: SinkingPass(), PMW);
2481 addFunctionPass(Pass: AMDGPULateCodeGenPreparePass(getTM()), PMW);
2482 }
2483
2484 // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
2485 // regions formed by them.
2486
2487 addFunctionPass(Pass: AMDGPUUnifyDivergentExitNodesPass(), PMW);
2488 addFunctionPass(Pass: FixIrreduciblePass(), PMW);
2489 addFunctionPass(Pass: UnifyLoopExitsPass(), PMW);
2490 addFunctionPass(Pass: StructurizeCFGPass(/*SkipUniformRegions=*/false), PMW);
2491
2492 addFunctionPass(Pass: AMDGPUAnnotateUniformValuesPass(), PMW);
2493
2494 addFunctionPass(Pass: SIAnnotateControlFlowPass(getTM()), PMW);
2495
2496 // TODO: Move this right after structurizeCFG to avoid extra divergence
2497 // analysis. This depends on stopping SIAnnotateControlFlow from making
2498 // control flow modifications.
2499 addFunctionPass(Pass: AMDGPURewriteUndefForPHIPass(), PMW);
2500
2501 if (getCGPassBuilderOption().EnableGlobalISelOption !=
2502 cl::boolOrDefault::BOU_TRUE ||
2503 !isGlobalISelAbortEnabled())
2504 addFunctionPass(Pass: LCSSAPass(), PMW);
2505
2506 if (TM.getOptLevel() > CodeGenOptLevel::Less) {
2507 flushFPMsToMPM(PMW);
2508 addModulePass(Pass: AMDGPUPerfHintAnalysisPass(getTM()), PMW);
2509 }
2510}
2511
2512void AMDGPUCodeGenPassBuilder::addILPOpts(PassManagerWrapper &PMW) {
2513 if (EnableEarlyIfConversion)
2514 addMachineFunctionPass(Pass: EarlyIfConverterPass(), PMW);
2515
2516 Base::addILPOpts(PMW);
2517}
2518
2519void AMDGPUCodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
2520 addModulePass(Pass: AMDGPUAsmPrinterBeginPass(), PMW,
2521 /*Force=*/true);
2522}
2523
2524void AMDGPUCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
2525 addMachineFunctionPass(Pass: AMDGPUAsmPrinterPass(), PMW);
2526}
2527
2528void AMDGPUCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
2529 addModulePass(Pass: AMDGPUAsmPrinterEndPass(), PMW);
2530}
2531
2532Error AMDGPUCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
2533 addMachineFunctionPass(Pass: AMDGPUISelDAGToDAGPass(TM), PMW);
2534 addMachineFunctionPass(Pass: SIFixSGPRCopiesPass(), PMW);
2535 addMachineFunctionPass(Pass: SILowerI1CopiesPass(), PMW);
2536 return Error::success();
2537}
2538
2539void AMDGPUCodeGenPassBuilder::addPreRewrite(PassManagerWrapper &PMW) {
2540 if (EnableRegReassign) {
2541 addMachineFunctionPass(Pass: GCNNSAReassignPass(), PMW);
2542 }
2543
2544 addMachineFunctionPass(Pass: AMDGPURewriteAGPRCopyMFMAPass(), PMW);
2545}
2546
2547void AMDGPUCodeGenPassBuilder::addMachineSSAOptimization(
2548 PassManagerWrapper &PMW) {
2549 Base::addMachineSSAOptimization(PMW);
2550
2551 addMachineFunctionPass(Pass: SIFoldOperandsPass(), PMW);
2552 if (EnableDPPCombine) {
2553 addMachineFunctionPass(Pass: GCNDPPCombinePass(), PMW);
2554 }
2555 addMachineFunctionPass(Pass: SILoadStoreOptimizerPass(), PMW);
2556 if (isPassEnabled(Opt: EnableSDWAPeephole)) {
2557 addMachineFunctionPass(Pass: SIPeepholeSDWAPass(), PMW);
2558 addMachineFunctionPass(Pass: EarlyMachineLICMPass(), PMW);
2559 addMachineFunctionPass(Pass: MachineCSEPass(), PMW);
2560 addMachineFunctionPass(Pass: SIFoldOperandsPass(), PMW);
2561 }
2562 addMachineFunctionPass(Pass: DeadMachineInstructionElimPass(), PMW);
2563 addMachineFunctionPass(Pass: SIShrinkInstructionsPass(), PMW);
2564}
2565
2566Error AMDGPUCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
2567 insertPass<PHIEliminationPass>(Pass: SILowerControlFlowPass());
2568
2569 insertPass<TwoAddressInstructionPass>(Pass: SIWholeQuadModePass());
2570
2571 return Base::addFastRegAlloc(PMW);
2572}
2573
2574Error AMDGPUCodeGenPassBuilder::addRegAssignAndRewriteFast(
2575 PassManagerWrapper &PMW) {
2576 if (auto Err = validateRegAllocOptions())
2577 return Err;
2578
2579 addMachineFunctionPass(Pass: GCNPreRALongBranchRegPass(), PMW);
2580
2581 // SGPR allocation - default to fast at -O0.
2582 if (SGPRRegAllocNPM == RegAllocType::Greedy)
2583 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateSGPRs, "sgpr"}), PMW);
2584 else
2585 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateSGPRs, "sgpr", false}),
2586 PMW);
2587
2588 // Equivalent of PEI for SGPRs.
2589 addMachineFunctionPass(Pass: SILowerSGPRSpillsPass(), PMW);
2590
2591 // To Allocate wwm registers used in whole quad mode operations (for shaders).
2592 addMachineFunctionPass(Pass: SIPreAllocateWWMRegsPass(), PMW);
2593
2594 // WWM allocation - default to fast at -O0.
2595 if (WWMRegAllocNPM == RegAllocType::Greedy)
2596 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateWWMRegs, "wwm"}), PMW);
2597 else
2598 addMachineFunctionPass(
2599 Pass: RegAllocFastPass({onlyAllocateWWMRegs, "wwm", false}), PMW);
2600
2601 addMachineFunctionPass(Pass: SILowerWWMCopiesPass(), PMW);
2602 addMachineFunctionPass(Pass: AMDGPUReserveWWMRegsPass(), PMW);
2603
2604 // VGPR allocation - default to fast at -O0.
2605 if (VGPRRegAllocNPM == RegAllocType::Greedy)
2606 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2607 else
2608 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2609
2610 return Error::success();
2611}
2612
2613Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
2614 if (EnableDCEInRA)
2615 insertPass<DetectDeadLanesPass>(Pass: DeadMachineInstructionElimPass());
2616
2617 // FIXME: when an instruction has a Killed operand, and the instruction is
2618 // inside a bundle, seems only the BUNDLE instruction appears as the Kills of
2619 // the register in LiveVariables, this would trigger a failure in verifier,
2620 // we should fix it and enable the verifier.
2621 if (OptVGPRLiveRange)
2622 insertPass<RequireAnalysisPass<LiveVariablesAnalysis, MachineFunction>>(
2623 Pass: SIOptimizeVGPRLiveRangePass());
2624
2625 // This must be run immediately after phi elimination and before
2626 // TwoAddressInstructions, otherwise the processing of the tied operand of
2627 // SI_ELSE will introduce a copy of the tied operand source after the else.
2628 insertPass<PHIEliminationPass>(Pass: SILowerControlFlowPass());
2629
2630 if (EnableRewritePartialRegUses)
2631 insertPass<RenameIndependentSubregsPass>(Pass: GCNRewritePartialRegUsesPass());
2632
2633 if (isPassEnabled(Opt: EnablePreRAOptimizations))
2634 insertPass<MachineSchedulerPass>(Pass: GCNPreRAOptimizationsPass());
2635
2636 // Allow the scheduler to run before SIWholeQuadMode inserts exec manipulation
2637 // instructions that cause scheduling barriers.
2638 insertPass<MachineSchedulerPass>(Pass: SIWholeQuadModePass());
2639
2640 if (OptExecMaskPreRA)
2641 insertPass<MachineSchedulerPass>(Pass: SIOptimizeExecMaskingPreRAPass());
2642
2643 // This is not an essential optimization and it has a noticeable impact on
2644 // compilation time, so we only enable it from O2.
2645 if (TM.getOptLevel() > CodeGenOptLevel::Less)
2646 insertPass<MachineSchedulerPass>(Pass: SIFormMemoryClausesPass());
2647
2648 return Base::addOptimizedRegAlloc(PMW);
2649}
2650
2651void AMDGPUCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
2652 if (getOptLevel() != CodeGenOptLevel::None)
2653 addMachineFunctionPass(Pass: AMDGPUPrepareAGPRAllocPass(), PMW);
2654}
2655
2656Expected<bool> AMDGPUCodeGenPassBuilder::addRegAssignAndRewriteOptimized(
2657 PassManagerWrapper &PMW) {
2658 if (auto Err = validateRegAllocOptions())
2659 return Err;
2660
2661 addMachineFunctionPass(Pass: GCNPreRALongBranchRegPass(), PMW);
2662
2663 // SGPR allocation - default to greedy at -O1 and above.
2664 if (SGPRRegAllocNPM == RegAllocType::Fast)
2665 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateSGPRs, "sgpr", false}),
2666 PMW);
2667 else
2668 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateSGPRs, "sgpr"}), PMW);
2669
2670 // Commit allocated register changes. This is mostly necessary because too
2671 // many things rely on the use lists of the physical registers, such as the
2672 // verifier. This is only necessary with allocators which use LiveIntervals,
2673 // since FastRegAlloc does the replacements itself.
2674 addMachineFunctionPass(Pass: VirtRegRewriterPass(false), PMW);
2675
2676 // At this point, the sgpr-regalloc has been done and it is good to have the
2677 // stack slot coloring to try to optimize the SGPR spill stack indices before
2678 // attempting the custom SGPR spill lowering.
2679 addMachineFunctionPass(Pass: StackSlotColoringPass(), PMW);
2680
2681 // Equivalent of PEI for SGPRs.
2682 addMachineFunctionPass(Pass: SILowerSGPRSpillsPass(), PMW);
2683
2684 // To Allocate wwm registers used in whole quad mode operations (for shaders).
2685 addMachineFunctionPass(Pass: SIPreAllocateWWMRegsPass(), PMW);
2686
2687 // WWM allocation - default to greedy at -O1 and above.
2688 if (WWMRegAllocNPM == RegAllocType::Fast)
2689 addMachineFunctionPass(
2690 Pass: RegAllocFastPass({onlyAllocateWWMRegs, "wwm", false}), PMW);
2691 else
2692 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateWWMRegs, "wwm"}), PMW);
2693 addMachineFunctionPass(Pass: SILowerWWMCopiesPass(), PMW);
2694 addMachineFunctionPass(Pass: VirtRegRewriterPass(false), PMW);
2695 addMachineFunctionPass(Pass: AMDGPUReserveWWMRegsPass(), PMW);
2696
2697 // VGPR allocation - default to greedy at -O1 and above.
2698 if (VGPRRegAllocNPM == RegAllocType::Fast)
2699 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2700 else
2701 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2702
2703 addPreRewrite(PMW);
2704 addMachineFunctionPass(Pass: VirtRegRewriterPass(true), PMW);
2705
2706 addMachineFunctionPass(Pass: AMDGPUMarkLastScratchLoadPass(), PMW);
2707 return true;
2708}
2709
2710void AMDGPUCodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) {
2711 addMachineFunctionPass(Pass: SIFixVGPRCopiesPass(), PMW);
2712 if (TM.getOptLevel() > CodeGenOptLevel::None)
2713 addMachineFunctionPass(Pass: SIOptimizeExecMaskingPass(), PMW);
2714 Base::addPostRegAlloc(PMW);
2715}
2716
2717void AMDGPUCodeGenPassBuilder::addPreSched2(PassManagerWrapper &PMW) {
2718 if (TM.getOptLevel() > CodeGenOptLevel::None)
2719 addMachineFunctionPass(Pass: SIShrinkInstructionsPass(), PMW);
2720 addMachineFunctionPass(Pass: SIPostRABundlerPass(), PMW);
2721}
2722
2723void AMDGPUCodeGenPassBuilder::addPostBBSections(PassManagerWrapper &PMW) {
2724 // We run this later to avoid passes like livedebugvalues and BBSections
2725 // having to deal with the apparent multi-entry functions we may generate.
2726 addMachineFunctionPass(Pass: AMDGPUPreloadKernArgPrologPass(), PMW);
2727}
2728
2729void AMDGPUCodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
2730 if (isPassEnabled(Opt: EnableVOPD, Level: CodeGenOptLevel::Less)) {
2731 addMachineFunctionPass(Pass: GCNCreateVOPDPass(), PMW);
2732 }
2733
2734 addMachineFunctionPass(Pass: SIMemoryLegalizerPass(), PMW);
2735 addMachineFunctionPass(Pass: SIInsertWaitcntsPass(), PMW);
2736
2737 addMachineFunctionPass(Pass: SIModeRegisterPass(), PMW);
2738
2739 if (TM.getOptLevel() > CodeGenOptLevel::None)
2740 addMachineFunctionPass(Pass: SIInsertHardClausesPass(), PMW);
2741
2742 addMachineFunctionPass(Pass: SILateBranchLoweringPass(), PMW);
2743
2744 if (isPassEnabled(Opt: EnableSetWavePriority, Level: CodeGenOptLevel::Less))
2745 addMachineFunctionPass(Pass: AMDGPUSetWavePriorityPass(), PMW);
2746
2747 if (TM.getOptLevel() > CodeGenOptLevel::None)
2748 addMachineFunctionPass(Pass: SIPreEmitPeepholePass(), PMW);
2749
2750 // The hazard recognizer that runs as part of the post-ra scheduler does not
2751 // guarantee to be able handle all hazards correctly. This is because if there
2752 // are multiple scheduling regions in a basic block, the regions are scheduled
2753 // bottom up, so when we begin to schedule a region we don't know what
2754 // instructions were emitted directly before it.
2755 //
2756 // Here we add a stand-alone hazard recognizer pass which can handle all
2757 // cases.
2758 addMachineFunctionPass(Pass: PostRAHazardRecognizerPass(), PMW);
2759 addMachineFunctionPass(Pass: AMDGPUWaitSGPRHazardsPass(), PMW);
2760 addMachineFunctionPass(Pass: AMDGPULowerVGPREncodingPass(), PMW);
2761
2762 if (isPassEnabled(Opt: EnableInsertDelayAlu, Level: CodeGenOptLevel::Less)) {
2763 addMachineFunctionPass(Pass: AMDGPUInsertDelayAluPass(), PMW);
2764 }
2765
2766 addMachineFunctionPass(Pass: BranchRelaxationPass(), PMW);
2767}
2768
2769bool AMDGPUCodeGenPassBuilder::isPassEnabled(const cl::opt<bool> &Opt,
2770 CodeGenOptLevel Level) const {
2771 if (Opt.getNumOccurrences())
2772 return Opt;
2773 if (TM.getOptLevel() < Level)
2774 return false;
2775 return Opt;
2776}
2777
2778void AMDGPUCodeGenPassBuilder::addEarlyCSEOrGVNPass(PassManagerWrapper &PMW) {
2779 if (TM.getOptLevel() == CodeGenOptLevel::Aggressive)
2780 addFunctionPass(Pass: GVNPass(), PMW);
2781 else
2782 addFunctionPass(Pass: EarlyCSEPass(), PMW);
2783}
2784
2785void AMDGPUCodeGenPassBuilder::addStraightLineScalarOptimizationPasses(
2786 PassManagerWrapper &PMW) {
2787 if (isPassEnabled(Opt: EnableLoopPrefetch, Level: CodeGenOptLevel::Aggressive))
2788 addFunctionPass(Pass: LoopDataPrefetchPass(), PMW);
2789
2790 addFunctionPass(Pass: SeparateConstOffsetFromGEPPass(), PMW);
2791
2792 // ReassociateGEPs exposes more opportunities for SLSR. See
2793 // the example in reassociate-geps-and-slsr.ll.
2794 addFunctionPass(Pass: StraightLineStrengthReducePass(), PMW);
2795
2796 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or
2797 // EarlyCSE can reuse.
2798 addEarlyCSEOrGVNPass(PMW);
2799
2800 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
2801 addFunctionPass(Pass: NaryReassociatePass(), PMW);
2802
2803 // NaryReassociate on GEPs creates redundant common expressions, so run
2804 // EarlyCSE after it.
2805 addFunctionPass(Pass: EarlyCSEPass(), PMW);
2806}
2807