1//===-- AMDGPUTargetMachine.cpp - TargetMachine for hw codegen targets-----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file contains both AMDGPU target machine and the CodeGen pass builder.
11/// The AMDGPU target machine contains all of the hardware specific information
12/// needed to emit code for SI+ GPUs in the legacy pass manager pipeline. The
13/// CodeGen pass builder handles the pass pipeline for new pass manager.
14//
15//===----------------------------------------------------------------------===//
16
17#include "AMDGPUTargetMachine.h"
18#include "AMDGPU.h"
19#include "AMDGPUAliasAnalysis.h"
20#include "AMDGPUAsmPrinter.h"
21#include "AMDGPUBarrierLatency.h"
22#include "AMDGPUCoExecSchedStrategy.h"
23#include "AMDGPUCtorDtorLowering.h"
24#include "AMDGPUExportClustering.h"
25#include "AMDGPUExportKernelRuntimeHandles.h"
26#include "AMDGPUHazardLatency.h"
27#include "AMDGPUIGroupLP.h"
28#include "AMDGPUISelDAGToDAG.h"
29#include "AMDGPULowerVGPREncoding.h"
30#include "AMDGPUMacroFusion.h"
31#include "AMDGPUNextUseAnalysis.h"
32#include "AMDGPUPerfHintAnalysis.h"
33#include "AMDGPUPreloadKernArgProlog.h"
34#include "AMDGPUPrepareAGPRAlloc.h"
35#include "AMDGPURemoveIncompatibleFunctions.h"
36#include "AMDGPUReserveWWMRegs.h"
37#include "AMDGPUResourceUsageAnalysis.h"
38#include "AMDGPUSplitModule.h"
39#include "AMDGPUTargetObjectFile.h"
40#include "AMDGPUTargetTransformInfo.h"
41#include "AMDGPUUnifyDivergentExitNodes.h"
42#include "AMDGPUWaitSGPRHazards.h"
43#include "GCNDPPCombine.h"
44#include "GCNIterativeScheduler.h"
45#include "GCNNSAReassign.h"
46#include "GCNPreRALongBranchReg.h"
47#include "GCNPreRAOptimizations.h"
48#include "GCNRewritePartialRegUses.h"
49#include "GCNSchedStrategy.h"
50#include "GCNVOPDUtils.h"
51#include "R600.h"
52#include "R600TargetMachine.h"
53#include "SIFixSGPRCopies.h"
54#include "SIFixVGPRCopies.h"
55#include "SIFoldOperands.h"
56#include "SIFormMemoryClauses.h"
57#include "SILoadStoreOptimizer.h"
58#include "SILowerControlFlow.h"
59#include "SILowerSGPRSpills.h"
60#include "SILowerWWMCopies.h"
61#include "SIMachineFunctionInfo.h"
62#include "SIMachineScheduler.h"
63#include "SIOptimizeExecMasking.h"
64#include "SIOptimizeExecMaskingPreRA.h"
65#include "SIOptimizeVGPRLiveRange.h"
66#include "SIPeepholeSDWA.h"
67#include "SIPostRABundler.h"
68#include "SIPreAllocateWWMRegs.h"
69#include "SIShrinkInstructions.h"
70#include "SIWholeQuadMode.h"
71#include "TargetInfo/AMDGPUTargetInfo.h"
72#include "Utils/AMDGPUBaseInfo.h"
73#include "llvm/Analysis/CGSCCPassManager.h"
74#include "llvm/Analysis/KernelInfo.h"
75#include "llvm/CodeGen/AtomicExpand.h"
76#include "llvm/CodeGen/BranchRelaxation.h"
77#include "llvm/CodeGen/DeadMachineInstructionElim.h"
78#include "llvm/CodeGen/DetectDeadLanes.h"
79#include "llvm/CodeGen/EarlyIfConversion.h"
80#include "llvm/CodeGen/FuncletLayout.h"
81#include "llvm/CodeGen/GCMetadata.h"
82#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
83#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
84#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
85#include "llvm/CodeGen/GlobalISel/Legalizer.h"
86#include "llvm/CodeGen/GlobalISel/Localizer.h"
87#include "llvm/CodeGen/MIRParser/MIParser.h"
88#include "llvm/CodeGen/MachineCSE.h"
89#include "llvm/CodeGen/MachineLICM.h"
90#include "llvm/CodeGen/MachinePipeliner.h"
91#include "llvm/CodeGen/MachineScheduler.h"
92#include "llvm/CodeGen/PHIElimination.h"
93#include "llvm/CodeGen/Passes.h"
94#include "llvm/CodeGen/PatchableFunction.h"
95#include "llvm/CodeGen/PostRAHazardRecognizer.h"
96#include "llvm/CodeGen/RegAllocGreedyPass.h"
97#include "llvm/CodeGen/RegAllocRegistry.h"
98#include "llvm/CodeGen/RenameIndependentSubregs.h"
99#include "llvm/CodeGen/ShadowStackGCLowering.h"
100#include "llvm/CodeGen/StackSlotColoring.h"
101#include "llvm/CodeGen/TargetPassConfig.h"
102#include "llvm/CodeGen/TwoAddressInstructionPass.h"
103#include "llvm/IR/DiagnosticInfo.h"
104#include "llvm/IR/IntrinsicsAMDGPU.h"
105#include "llvm/IR/Module.h"
106#include "llvm/IR/PassManager.h"
107#include "llvm/IR/PatternMatch.h"
108#include "llvm/InitializePasses.h"
109#include "llvm/MC/TargetRegistry.h"
110#include "llvm/Passes/CodeGenPassBuilder.h"
111#include "llvm/Passes/PassBuilder.h"
112#include "llvm/Support/Compiler.h"
113#include "llvm/Support/FormatVariadic.h"
114#include "llvm/TargetParser/AMDGPUTargetParser.h"
115#include "llvm/Transforms/HipStdPar/HipStdPar.h"
116#include "llvm/Transforms/IPO.h"
117#include "llvm/Transforms/IPO/AlwaysInliner.h"
118#include "llvm/Transforms/IPO/ExpandVariadics.h"
119#include "llvm/Transforms/IPO/GlobalDCE.h"
120#include "llvm/Transforms/IPO/Internalize.h"
121#include "llvm/Transforms/Scalar.h"
122#include "llvm/Transforms/Scalar/EarlyCSE.h"
123#include "llvm/Transforms/Scalar/FlattenCFG.h"
124#include "llvm/Transforms/Scalar/GVN.h"
125#include "llvm/Transforms/Scalar/InferAddressSpaces.h"
126#include "llvm/Transforms/Scalar/LICM.h"
127#include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
128#include "llvm/Transforms/Scalar/LoopPassManager.h"
129#include "llvm/Transforms/Scalar/NaryReassociate.h"
130#include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"
131#include "llvm/Transforms/Scalar/Sink.h"
132#include "llvm/Transforms/Scalar/StraightLineStrengthReduce.h"
133#include "llvm/Transforms/Scalar/StructurizeCFG.h"
134#include "llvm/Transforms/Utils.h"
135#include "llvm/Transforms/Utils/FixIrreducible.h"
136#include "llvm/Transforms/Utils/LCSSA.h"
137#include "llvm/Transforms/Utils/LowerSwitch.h"
138#include "llvm/Transforms/Utils/SimplifyLibCalls.h"
139#include "llvm/Transforms/Utils/UnifyLoopExits.h"
140#include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
141#include <optional>
142
143using namespace llvm;
144using namespace llvm::PatternMatch;
145
146namespace {
147//===----------------------------------------------------------------------===//
148// AMDGPU CodeGen Pass Builder interface.
149//===----------------------------------------------------------------------===//
150
151class AMDGPUCodeGenPassBuilder : public CodeGenPassBuilder {
152 using Base = CodeGenPassBuilder;
153
154 GCNTargetMachine &getTM() const {
155 return static_cast<GCNTargetMachine &>(TM);
156 }
157
158public:
159 AMDGPUCodeGenPassBuilder(GCNTargetMachine &TM,
160 const CGPassBuilderOption &Opts,
161 PassInstrumentationCallbacks *PIC);
162
163 void addIRPasses(PassManagerWrapper &PMW) override;
164 void addCodeGenPrepare(PassManagerWrapper &PMW) override;
165 void addPreISel(PassManagerWrapper &PMW) override;
166 void addILPOpts(PassManagerWrapper &PMW) override;
167 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
168 void addAsmPrinter(PassManagerWrapper &PMW) override;
169 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
170 Error addInstSelector(PassManagerWrapper &PMW) override;
171 Error addIRTranslator(PassManagerWrapper &PMW) override;
172 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
173 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
174 void addPreRegBankSelect(PassManagerWrapper &PMW) override;
175 Error addRegBankSelect(PassManagerWrapper &PMW) override;
176 void addPreGlobalInstructionSelect(PassManagerWrapper &PMW) override;
177 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
178 void addPreRewrite(PassManagerWrapper &PMW) override;
179 void addMachineSSAOptimization(PassManagerWrapper &PMW) override;
180 void addPostRegAlloc(PassManagerWrapper &PMW) override;
181 void addPreEmitPass(PassManagerWrapper &PMW) override;
182 Error addRegAssignAndRewriteFast(PassManagerWrapper &PMW) override;
183 Expected<bool>
184 addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) override;
185 void addPreRegAlloc(PassManagerWrapper &PMW) override;
186 Error addFastRegAlloc(PassManagerWrapper &PMW) override;
187 Error addOptimizedRegAlloc(PassManagerWrapper &PMW) override;
188 void addPreSched2(PassManagerWrapper &PMW) override;
189 void addPostBBSections(PassManagerWrapper &PMW) override;
190
191private:
192 Error validateRegAllocOptions() const;
193
194public:
195 /// Check if a pass is enabled given \p Opt option. The option always
196 /// overrides defaults if explicitly used. Otherwise its default will be used
197 /// given that a pass shall work at an optimization \p Level minimum.
198 bool isPassEnabled(const cl::opt<bool> &Opt,
199 CodeGenOptLevel Level = CodeGenOptLevel::Default) const;
200 void addEarlyCSEOrGVNPass(PassManagerWrapper &PMW);
201 void addStraightLineScalarOptimizationPasses(PassManagerWrapper &PMW);
202};
203
204class SGPRRegisterRegAlloc : public RegisterRegAllocBase<SGPRRegisterRegAlloc> {
205public:
206 SGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
207 : RegisterRegAllocBase(N, D, C) {}
208};
209
210class VGPRRegisterRegAlloc : public RegisterRegAllocBase<VGPRRegisterRegAlloc> {
211public:
212 VGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
213 : RegisterRegAllocBase(N, D, C) {}
214};
215
216class WWMRegisterRegAlloc : public RegisterRegAllocBase<WWMRegisterRegAlloc> {
217public:
218 WWMRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
219 : RegisterRegAllocBase(N, D, C) {}
220};
221
222static bool onlyAllocateSGPRs(const TargetRegisterInfo &TRI,
223 const MachineRegisterInfo &MRI,
224 const Register Reg) {
225 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
226 return static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC);
227}
228
229static bool onlyAllocateVGPRs(const TargetRegisterInfo &TRI,
230 const MachineRegisterInfo &MRI,
231 const Register Reg) {
232 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
233 return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC);
234}
235
236static bool onlyAllocateWWMRegs(const TargetRegisterInfo &TRI,
237 const MachineRegisterInfo &MRI,
238 const Register Reg) {
239 const SIMachineFunctionInfo *MFI =
240 MRI.getMF().getInfo<SIMachineFunctionInfo>();
241 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
242 return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC) &&
243 MFI->checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG);
244}
245
246/// -{sgpr|wwm|vgpr}-regalloc=... command line option.
247static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
248
249/// A dummy default pass factory indicates whether the register allocator is
250/// overridden on the command line.
251static llvm::once_flag InitializeDefaultSGPRRegisterAllocatorFlag;
252static llvm::once_flag InitializeDefaultVGPRRegisterAllocatorFlag;
253static llvm::once_flag InitializeDefaultWWMRegisterAllocatorFlag;
254
255static SGPRRegisterRegAlloc
256defaultSGPRRegAlloc("default",
257 "pick SGPR register allocator based on -O option",
258 useDefaultRegisterAllocator);
259
260static cl::opt<SGPRRegisterRegAlloc::FunctionPassCtor, false,
261 RegisterPassParser<SGPRRegisterRegAlloc>>
262SGPRRegAlloc("sgpr-regalloc", cl::Hidden, cl::init(Val: &useDefaultRegisterAllocator),
263 cl::desc("Register allocator to use for SGPRs"));
264
265static cl::opt<VGPRRegisterRegAlloc::FunctionPassCtor, false,
266 RegisterPassParser<VGPRRegisterRegAlloc>>
267VGPRRegAlloc("vgpr-regalloc", cl::Hidden, cl::init(Val: &useDefaultRegisterAllocator),
268 cl::desc("Register allocator to use for VGPRs"));
269
270static cl::opt<WWMRegisterRegAlloc::FunctionPassCtor, false,
271 RegisterPassParser<WWMRegisterRegAlloc>>
272 WWMRegAlloc("wwm-regalloc", cl::Hidden,
273 cl::init(Val: &useDefaultRegisterAllocator),
274 cl::desc("Register allocator to use for WWM registers"));
275
276// New pass manager register allocator options for AMDGPU
277static cl::opt<RegAllocType, false, RegAllocTypeParser> SGPRRegAllocNPM(
278 "sgpr-regalloc-npm", cl::Hidden, cl::init(Val: RegAllocType::Default),
279 cl::desc("Register allocator for SGPRs (new pass manager)"));
280
281static cl::opt<RegAllocType, false, RegAllocTypeParser> VGPRRegAllocNPM(
282 "vgpr-regalloc-npm", cl::Hidden, cl::init(Val: RegAllocType::Default),
283 cl::desc("Register allocator for VGPRs (new pass manager)"));
284
285static cl::opt<RegAllocType, false, RegAllocTypeParser> WWMRegAllocNPM(
286 "wwm-regalloc-npm", cl::Hidden, cl::init(Val: RegAllocType::Default),
287 cl::desc("Register allocator for WWM registers (new pass manager)"));
288
289/// Check if the given RegAllocType is supported for AMDGPU NPM register
290/// allocation. Only Fast and Greedy are supported; Basic and PBQP are not.
291static Error checkRegAllocSupported(RegAllocType RAType, StringRef RegName) {
292 if (RAType == RegAllocType::Basic || RAType == RegAllocType::PBQP) {
293 return make_error<StringError>(
294 Args: Twine("unsupported register allocator '") +
295 (RAType == RegAllocType::Basic ? "basic" : "pbqp") + "' for " +
296 RegName + " registers",
297 Args: inconvertibleErrorCode());
298 }
299 return Error::success();
300}
301
302Error AMDGPUCodeGenPassBuilder::validateRegAllocOptions() const {
303 // 1. Generic --regalloc-npm is not supported for AMDGPU.
304 if (Opt.RegAlloc != RegAllocType::Unset) {
305 return make_error<StringError>(
306 Args: "-regalloc-npm not supported for amdgcn. Use -sgpr-regalloc-npm, "
307 "-vgpr-regalloc-npm, and -wwm-regalloc-npm",
308 Args: inconvertibleErrorCode());
309 }
310
311 // 2. Legacy PM regalloc options are not compatible with NPM.
312 if (SGPRRegAlloc.getNumOccurrences() > 0 ||
313 VGPRRegAlloc.getNumOccurrences() > 0 ||
314 WWMRegAlloc.getNumOccurrences() > 0) {
315 return make_error<StringError>(
316 Args: "-sgpr-regalloc, -vgpr-regalloc, and -wwm-regalloc are legacy PM "
317 "options. Use -sgpr-regalloc-npm, -vgpr-regalloc-npm, and "
318 "-wwm-regalloc-npm with the new pass manager",
319 Args: inconvertibleErrorCode());
320 }
321
322 // 3. Only Fast and Greedy allocators are supported for AMDGPU.
323 if (auto Err = checkRegAllocSupported(RAType: SGPRRegAllocNPM, RegName: "SGPR"))
324 return Err;
325 if (auto Err = checkRegAllocSupported(RAType: WWMRegAllocNPM, RegName: "WWM"))
326 return Err;
327 if (auto Err = checkRegAllocSupported(RAType: VGPRRegAllocNPM, RegName: "VGPR"))
328 return Err;
329
330 return Error::success();
331}
332
333static void initializeDefaultSGPRRegisterAllocatorOnce() {
334 RegisterRegAlloc::FunctionPassCtor Ctor = SGPRRegisterRegAlloc::getDefault();
335
336 if (!Ctor) {
337 Ctor = SGPRRegAlloc;
338 SGPRRegisterRegAlloc::setDefault(SGPRRegAlloc);
339 }
340}
341
342static void initializeDefaultVGPRRegisterAllocatorOnce() {
343 RegisterRegAlloc::FunctionPassCtor Ctor = VGPRRegisterRegAlloc::getDefault();
344
345 if (!Ctor) {
346 Ctor = VGPRRegAlloc;
347 VGPRRegisterRegAlloc::setDefault(VGPRRegAlloc);
348 }
349}
350
351static void initializeDefaultWWMRegisterAllocatorOnce() {
352 RegisterRegAlloc::FunctionPassCtor Ctor = WWMRegisterRegAlloc::getDefault();
353
354 if (!Ctor) {
355 Ctor = WWMRegAlloc;
356 WWMRegisterRegAlloc::setDefault(WWMRegAlloc);
357 }
358}
359
360static FunctionPass *createBasicSGPRRegisterAllocator() {
361 return createBasicRegisterAllocator(F: onlyAllocateSGPRs);
362}
363
364static FunctionPass *createGreedySGPRRegisterAllocator() {
365 return createGreedyRegisterAllocator(F: onlyAllocateSGPRs);
366}
367
368static FunctionPass *createFastSGPRRegisterAllocator() {
369 return createFastRegisterAllocator(F: onlyAllocateSGPRs, ClearVirtRegs: false);
370}
371
372static FunctionPass *createBasicVGPRRegisterAllocator() {
373 return createBasicRegisterAllocator(F: onlyAllocateVGPRs);
374}
375
376static FunctionPass *createGreedyVGPRRegisterAllocator() {
377 return createGreedyRegisterAllocator(F: onlyAllocateVGPRs);
378}
379
380static FunctionPass *createFastVGPRRegisterAllocator() {
381 return createFastRegisterAllocator(F: onlyAllocateVGPRs, ClearVirtRegs: true);
382}
383
384static FunctionPass *createBasicWWMRegisterAllocator() {
385 return createBasicRegisterAllocator(F: onlyAllocateWWMRegs);
386}
387
388static FunctionPass *createGreedyWWMRegisterAllocator() {
389 return createGreedyRegisterAllocator(F: onlyAllocateWWMRegs);
390}
391
392static FunctionPass *createFastWWMRegisterAllocator() {
393 return createFastRegisterAllocator(F: onlyAllocateWWMRegs, ClearVirtRegs: false);
394}
395
396static SGPRRegisterRegAlloc basicRegAllocSGPR(
397 "basic", "basic register allocator", createBasicSGPRRegisterAllocator);
398static SGPRRegisterRegAlloc greedyRegAllocSGPR(
399 "greedy", "greedy register allocator", createGreedySGPRRegisterAllocator);
400
401static SGPRRegisterRegAlloc fastRegAllocSGPR(
402 "fast", "fast register allocator", createFastSGPRRegisterAllocator);
403
404
405static VGPRRegisterRegAlloc basicRegAllocVGPR(
406 "basic", "basic register allocator", createBasicVGPRRegisterAllocator);
407static VGPRRegisterRegAlloc greedyRegAllocVGPR(
408 "greedy", "greedy register allocator", createGreedyVGPRRegisterAllocator);
409
410static VGPRRegisterRegAlloc fastRegAllocVGPR(
411 "fast", "fast register allocator", createFastVGPRRegisterAllocator);
412static WWMRegisterRegAlloc basicRegAllocWWMReg("basic",
413 "basic register allocator",
414 createBasicWWMRegisterAllocator);
415static WWMRegisterRegAlloc
416 greedyRegAllocWWMReg("greedy", "greedy register allocator",
417 createGreedyWWMRegisterAllocator);
418static WWMRegisterRegAlloc fastRegAllocWWMReg("fast", "fast register allocator",
419 createFastWWMRegisterAllocator);
420
421static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {
422 return Phase == ThinOrFullLTOPhase::FullLTOPreLink ||
423 Phase == ThinOrFullLTOPhase::ThinLTOPreLink;
424}
425} // anonymous namespace
426
427static cl::opt<bool>
428EnableEarlyIfConversion("amdgpu-early-ifcvt", cl::Hidden,
429 cl::desc("Run early if-conversion"),
430 cl::init(Val: false));
431
432static cl::opt<bool>
433OptExecMaskPreRA("amdgpu-opt-exec-mask-pre-ra", cl::Hidden,
434 cl::desc("Run pre-RA exec mask optimizations"),
435 cl::init(Val: true));
436
437static cl::opt<bool>
438 LowerCtorDtor("amdgpu-lower-global-ctor-dtor",
439 cl::desc("Lower GPU ctor / dtors to globals on the device."),
440 cl::init(Val: true), cl::Hidden);
441
442// Option to disable vectorizer for tests.
443static cl::opt<bool> EnableLoadStoreVectorizer(
444 "amdgpu-load-store-vectorizer",
445 cl::desc("Enable load store vectorizer"),
446 cl::init(Val: true),
447 cl::Hidden);
448
449// Option to control global loads scalarization
450static cl::opt<bool> ScalarizeGlobal(
451 "amdgpu-scalarize-global-loads",
452 cl::desc("Enable global load scalarization"),
453 cl::init(Val: true),
454 cl::Hidden);
455
456// Option to run internalize pass.
457static cl::opt<bool> InternalizeSymbols(
458 "amdgpu-internalize-symbols",
459 cl::desc("Enable elimination of non-kernel functions and unused globals"),
460 cl::init(Val: false),
461 cl::Hidden);
462
463// Option to inline all early.
464static cl::opt<bool> EarlyInlineAll(
465 "amdgpu-early-inline-all",
466 cl::desc("Inline all functions early"),
467 cl::init(Val: false),
468 cl::Hidden);
469
470static cl::opt<bool> RemoveIncompatibleFunctions(
471 "amdgpu-enable-remove-incompatible-functions", cl::Hidden,
472 cl::desc("Enable removal of functions when they"
473 "use features not supported by the target GPU"),
474 cl::init(Val: true));
475
476static cl::opt<bool> EnableSDWAPeephole(
477 "amdgpu-sdwa-peephole",
478 cl::desc("Enable SDWA peepholer"),
479 cl::init(Val: true));
480
481static cl::opt<bool> EnableDPPCombine(
482 "amdgpu-dpp-combine",
483 cl::desc("Enable DPP combiner"),
484 cl::init(Val: true));
485
486// Enable address space based alias analysis
487static cl::opt<bool> EnableAMDGPUAliasAnalysis("enable-amdgpu-aa", cl::Hidden,
488 cl::desc("Enable AMDGPU Alias Analysis"),
489 cl::init(Val: true));
490
491static cl::opt<bool>
492 XnackSetting("amdgpu-xnack",
493 cl::desc("Force amdgpu.xnack value for testing"),
494 cl::ReallyHidden);
495
496static cl::opt<bool>
497 SramEccSetting("amdgpu-sramecc",
498 cl::desc("Force amdgpu.sramecc for testing"),
499 cl::ReallyHidden);
500
501// Enable lib calls simplifications
502static cl::opt<bool> EnableLibCallSimplify(
503 "amdgpu-simplify-libcall",
504 cl::desc("Enable amdgpu library simplifications"),
505 cl::init(Val: true),
506 cl::Hidden);
507
508static cl::opt<bool> EnableLowerKernelArguments(
509 "amdgpu-ir-lower-kernel-arguments",
510 cl::desc("Lower kernel argument loads in IR pass"),
511 cl::init(Val: true),
512 cl::Hidden);
513
514static cl::opt<bool> EnableRegReassign(
515 "amdgpu-reassign-regs",
516 cl::desc("Enable register reassign optimizations on gfx10+"),
517 cl::init(Val: true),
518 cl::Hidden);
519
520static cl::opt<bool> OptVGPRLiveRange(
521 "amdgpu-opt-vgpr-liverange",
522 cl::desc("Enable VGPR liverange optimizations for if-else structure"),
523 cl::init(Val: true), cl::Hidden);
524
525static cl::opt<ScanOptions> AMDGPUAtomicOptimizerStrategy(
526 "amdgpu-atomic-optimizer-strategy",
527 cl::desc("Select DPP or Iterative strategy for scan"),
528 cl::init(Val: ScanOptions::Iterative),
529 cl::values(
530 clEnumValN(ScanOptions::DPP, "DPP", "Use DPP operations for scan"),
531 clEnumValN(ScanOptions::Iterative, "Iterative",
532 "Use Iterative approach for scan"),
533 clEnumValN(ScanOptions::None, "None", "Disable atomic optimizer")));
534
535// Enable Mode register optimization
536static cl::opt<bool> EnableSIModeRegisterPass(
537 "amdgpu-mode-register",
538 cl::desc("Enable mode register pass"),
539 cl::init(Val: true),
540 cl::Hidden);
541
542// Enable GFX11+ s_delay_alu insertion
543static cl::opt<bool>
544 EnableInsertDelayAlu("amdgpu-enable-delay-alu",
545 cl::desc("Enable s_delay_alu insertion"),
546 cl::init(Val: true), cl::Hidden);
547
548// Enable GFX11+ VOPD
549static cl::opt<bool>
550 EnableVOPD("amdgpu-enable-vopd",
551 cl::desc("Enable VOPD, dual issue of VALU in wave32"),
552 cl::init(Val: true), cl::Hidden);
553
554// Option is used in lit tests to prevent deadcoding of patterns inspected.
555static cl::opt<bool>
556EnableDCEInRA("amdgpu-dce-in-ra",
557 cl::init(Val: true), cl::Hidden,
558 cl::desc("Enable machine DCE inside regalloc"));
559
560static cl::opt<bool> EnableSetWavePriority("amdgpu-set-wave-priority",
561 cl::desc("Adjust wave priority"),
562 cl::init(Val: false), cl::Hidden);
563
564static cl::opt<bool> EnableScalarIRPasses(
565 "amdgpu-scalar-ir-passes",
566 cl::desc("Enable scalar IR passes"),
567 cl::init(Val: true),
568 cl::Hidden);
569
570static cl::opt<bool> EnableLowerExecSync(
571 "amdgpu-enable-lower-exec-sync",
572 cl::desc("Enable lowering of execution synchronization."), cl::init(Val: true),
573 cl::Hidden);
574
575static cl::opt<bool>
576 EnableSwLowerLDS("amdgpu-enable-sw-lower-lds",
577 cl::desc("Enable lowering of lds to global memory pass "
578 "and asan instrument resulting IR."),
579 cl::init(Val: true), cl::Hidden);
580
581static cl::opt<bool, true> EnableObjectLinking(
582 "amdgpu-enable-object-linking",
583 cl::desc("Enable object linking for cross-TU LDS and ABI support"),
584 cl::location(L&: AMDGPUTargetMachine::EnableObjectLinking), cl::init(Val: false),
585 cl::Hidden);
586
587static cl::opt<bool, true> EnableLowerModuleLDS(
588 "amdgpu-enable-lower-module-lds", cl::desc("Enable lower module lds pass"),
589 cl::location(L&: AMDGPUTargetMachine::EnableLowerModuleLDS), cl::init(Val: true),
590 cl::Hidden);
591
592static cl::opt<bool> EnablePreRAOptimizations(
593 "amdgpu-enable-pre-ra-optimizations",
594 cl::desc("Enable Pre-RA optimizations pass"), cl::init(Val: true),
595 cl::Hidden);
596
597static cl::opt<bool> EnablePromoteKernelArguments(
598 "amdgpu-enable-promote-kernel-arguments",
599 cl::desc("Enable promotion of flat kernel pointer arguments to global"),
600 cl::Hidden, cl::init(Val: true));
601
602static cl::opt<bool> EnableImageIntrinsicOptimizer(
603 "amdgpu-enable-image-intrinsic-optimizer",
604 cl::desc("Enable image intrinsic optimizer pass"), cl::init(Val: true),
605 cl::Hidden);
606
607static cl::opt<bool>
608 EnableLoopPrefetch("amdgpu-loop-prefetch",
609 cl::desc("Enable loop data prefetch on AMDGPU"),
610 cl::Hidden, cl::init(Val: false));
611
612static cl::opt<std::string>
613 AMDGPUSchedStrategy("amdgpu-sched-strategy",
614 cl::desc("Select custom AMDGPU scheduling strategy."),
615 cl::Hidden, cl::init(Val: ""));
616
617// Scheduler selection is consulted both when creating the scheduler and from
618// overrideSchedPolicy(), so keep the attribute and global command line handling
619// in one helper.
620StringRef llvm::AMDGPU::getSchedStrategy(const Function &F) {
621 Attribute SchedStrategyAttr = F.getFnAttribute(Kind: "amdgpu-sched-strategy");
622 if (SchedStrategyAttr.isValid())
623 return SchedStrategyAttr.getValueAsString();
624
625 if (!AMDGPUSchedStrategy.empty())
626 return AMDGPUSchedStrategy;
627
628 return "";
629}
630
631static void
632diagnoseUnsupportedCoExecSchedulerSelection(const Function &F,
633 const GCNSubtarget &ST) {
634 if (ST.hasGFX1250Insts() || ST.hasGFX950Insts())
635 return;
636
637 F.getContext().diagnose(DI: DiagnosticInfoUnsupported(
638 F,
639 "'amdgpu-sched-strategy'='coexec' is only supported for gfx1250/gfx950",
640 DiagnosticLocation(), DS_Warning));
641}
642
643static bool useNoopPostScheduler(const Function &F) {
644 Attribute PostSchedStrategyAttr =
645 F.getFnAttribute(Kind: "amdgpu-post-sched-strategy");
646 return PostSchedStrategyAttr.isValid() &&
647 PostSchedStrategyAttr.getValueAsString() == "nop";
648}
649
650static cl::opt<bool> EnableRewritePartialRegUses(
651 "amdgpu-enable-rewrite-partial-reg-uses",
652 cl::desc("Enable rewrite partial reg uses pass"), cl::init(Val: true),
653 cl::Hidden);
654
655static cl::opt<bool> EnableHipStdPar(
656 "amdgpu-enable-hipstdpar",
657 cl::desc("Enable HIP Standard Parallelism Offload support"), cl::init(Val: false),
658 cl::Hidden);
659
660static cl::opt<bool>
661 EnableAMDGPUAttributor("amdgpu-attributor-enable",
662 cl::desc("Enable AMDGPUAttributorPass"),
663 cl::init(Val: true), cl::Hidden);
664
665static cl::opt<bool> HasClosedWorldAssumption(
666 "amdgpu-link-time-closed-world",
667 cl::desc("Whether has closed-world assumption at link time"),
668 cl::init(Val: false), cl::Hidden);
669
670static cl::opt<bool> EnableUniformIntrinsicCombine(
671 "amdgpu-enable-uniform-intrinsic-combine",
672 cl::desc("Enable/Disable the Uniform Intrinsic Combine Pass"),
673 cl::init(Val: true), cl::Hidden);
674
675static cl::opt<bool>
676 EnableMachinePipeliner("amdgpu-enable-pipeliner",
677 cl::desc("Enable Machine Pipeliner for AMDGCN"),
678 cl::init(Val: false), cl::Hidden);
679
680extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
681 // Register the target
682 RegisterTargetMachine<R600TargetMachine> X(getTheR600Target());
683 RegisterTargetMachine<GCNTargetMachine> Y(getTheGCNTarget());
684 RegisterTargetMachine<GCNTargetMachine> YLegacy(getTheGCNLegacyTarget());
685
686 PassRegistry *PR = PassRegistry::getPassRegistry();
687 initializeR600ClauseMergePassPass(*PR);
688 initializeR600ControlFlowFinalizerPass(*PR);
689 initializeR600PacketizerPass(*PR);
690 initializeR600ExpandSpecialInstrsPassPass(*PR);
691 initializeR600VectorRegMergerPass(*PR);
692 initializeR600EmitClauseMarkersPass(*PR);
693 initializeR600MachineCFGStructurizerPass(*PR);
694 initializeGlobalISel(*PR);
695 initializeAMDGPUAsmPrinterPass(*PR);
696 initializeAMDGPUDAGToDAGISelLegacyPass(*PR);
697 initializeAMDGPUPrepareAGPRAllocLegacyPass(*PR);
698 initializeGCNDPPCombineLegacyPass(*PR);
699 initializeSILowerI1CopiesLegacyPass(*PR);
700 initializeAMDGPUGlobalISelDivergenceLoweringLegacyPass(*PR);
701 initializeAMDGPURegBankSelectLegacyPass(*PR);
702 initializeAMDGPURegBankLegalizeLegacyPass(*PR);
703 initializeSILowerWWMCopiesLegacyPass(*PR);
704 initializeAMDGPUMarkLastScratchLoadLegacyPass(*PR);
705 initializeSILowerSGPRSpillsLegacyPass(*PR);
706 initializeSIFixSGPRCopiesLegacyPass(*PR);
707 initializeSIFixVGPRCopiesLegacyPass(*PR);
708 initializeSIFoldOperandsLegacyPass(*PR);
709 initializeSIPeepholeSDWALegacyPass(*PR);
710 initializeSIShrinkInstructionsLegacyPass(*PR);
711 initializeSIOptimizeExecMaskingPreRALegacyPass(*PR);
712 initializeSIOptimizeVGPRLiveRangeLegacyPass(*PR);
713 initializeAMDGPUNextUseAnalysisLegacyPassPass(*PR);
714 initializeAMDGPUNextUseAnalysisPrinterLegacyPassPass(*PR);
715 initializeSILoadStoreOptimizerLegacyPass(*PR);
716 initializeAMDGPUCtorDtorLoweringLegacyPass(*PR);
717 initializeAMDGPUAlwaysInlinePass(*PR);
718 initializeAMDGPULowerExecSyncLegacyPass(*PR);
719 initializeAMDGPUSwLowerLDSLegacyPass(*PR);
720 initializeAMDGPUAnnotateUniformValuesLegacyPass(*PR);
721 initializeAMDGPUAtomicOptimizerPass(*PR);
722 initializeAMDGPULowerKernelArgumentsPass(*PR);
723 initializeAMDGPUPromoteKernelArgumentsPass(*PR);
724 initializeAMDGPULowerKernelAttributesPass(*PR);
725 initializeAMDGPUExportKernelRuntimeHandlesLegacyPass(*PR);
726 initializeAMDGPUPostLegalizerCombinerLegacyPass(*PR);
727 initializeAMDGPUPreLegalizerCombinerLegacyPass(*PR);
728 initializeAMDGPURegBankCombinerLegacyPass(*PR);
729 initializeAMDGPUPromoteAllocaPass(*PR);
730 initializeAMDGPUCodeGenPreparePass(*PR);
731 initializeAMDGPULateCodeGenPrepareLegacyPass(*PR);
732 initializeAMDGPURemoveIncompatibleFunctionsLegacyPass(*PR);
733 initializeAMDGPULowerModuleLDSLegacyPass(*PR);
734 initializeAMDGPULowerBufferFatPointersPass(*PR);
735 initializeAMDGPULowerIntrinsicsLegacyPass(*PR);
736 initializeAMDGPUReserveWWMRegsLegacyPass(*PR);
737 initializeAMDGPURewriteAGPRCopyMFMALegacyPass(*PR);
738 initializeAMDGPURewriteOutArgumentsPass(*PR);
739 initializeAMDGPURewriteUndefForPHILegacyPass(*PR);
740 initializeSIAnnotateControlFlowLegacyPass(*PR);
741 initializeAMDGPUInsertDelayAluLegacyPass(*PR);
742 initializeAMDGPULowerVGPREncodingLegacyPass(*PR);
743 initializeSIInsertHardClausesLegacyPass(*PR);
744 initializeSIInsertWaitcntsLegacyPass(*PR);
745 initializeSIModeRegisterLegacyPass(*PR);
746 initializeSIWholeQuadModeLegacyPass(*PR);
747 initializeSILowerControlFlowLegacyPass(*PR);
748 initializeSIPreEmitPeepholeLegacyPass(*PR);
749 initializeSILateBranchLoweringLegacyPass(*PR);
750 initializeSIMemoryLegalizerLegacyPass(*PR);
751 initializeSIOptimizeExecMaskingLegacyPass(*PR);
752 initializeSIPreAllocateWWMRegsLegacyPass(*PR);
753 initializeSIFormMemoryClausesLegacyPass(*PR);
754 initializeSIPostRABundlerLegacyPass(*PR);
755 initializeGCNCreateVOPDLegacyPass(*PR);
756 initializeAMDGPUUnifyDivergentExitNodesLegacyPass(*PR);
757 initializeAMDGPUAAWrapperPassPass(*PR);
758 initializeAMDGPUExternalAAWrapperPass(*PR);
759 initializeAMDGPUImageIntrinsicOptimizerPass(*PR);
760 initializeAMDGPUPrintfRuntimeBindingPass(*PR);
761 initializeAMDGPUResourceUsageAnalysisWrapperPassPass(*PR);
762 initializeGCNNSAReassignLegacyPass(*PR);
763 initializeGCNPreRAOptimizationsLegacyPass(*PR);
764 initializeGCNPreRALongBranchRegLegacyPass(*PR);
765 initializeGCNRewritePartialRegUsesLegacyPass(*PR);
766 initializeGCNRegPressurePrinterPass(*PR);
767 initializeAMDGPUPreloadKernArgPrologLegacyPass(*PR);
768 initializeAMDGPUWaitSGPRHazardsLegacyPass(*PR);
769 initializeAMDGPUPreloadKernelArgumentsLegacyPass(*PR);
770 initializeAMDGPUUniformIntrinsicCombineLegacyPass(*PR);
771}
772
773static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
774 return std::make_unique<AMDGPUTargetObjectFile>();
775}
776
777static ScheduleDAGInstrs *createSIMachineScheduler(MachineSchedContext *C) {
778 return new SIScheduleDAGMI(C);
779}
780
781static ScheduleDAGInstrs *
782createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
783 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
784 ScheduleDAGMILive *DAG =
785 new GCNScheduleDAGMILive(C, std::make_unique<GCNMaxOccupancySchedStrategy>(args&: C));
786 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
787 if (ST.shouldClusterStores())
788 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
789 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
790 DAG->addMutation(Mutation: createAMDGPUMacroFusionDAGMutation());
791 DAG->addMutation(Mutation: createAMDGPUExportClusteringDAGMutation());
792 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
793 DAG->addMutation(Mutation: createAMDGPUHazardLatencyDAGMutation(MF: C->MF));
794 return DAG;
795}
796
797static ScheduleDAGInstrs *
798createGCNMaxILPMachineScheduler(MachineSchedContext *C) {
799 ScheduleDAGMILive *DAG =
800 new GCNScheduleDAGMILive(C, std::make_unique<GCNMaxILPSchedStrategy>(args&: C));
801 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
802 return DAG;
803}
804
805static ScheduleDAGInstrs *
806createGCNMaxMemoryClauseMachineScheduler(MachineSchedContext *C) {
807 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
808 ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive(
809 C, std::make_unique<GCNMaxMemoryClauseSchedStrategy>(args&: C));
810 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
811 if (ST.shouldClusterStores())
812 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
813 DAG->addMutation(Mutation: createAMDGPUExportClusteringDAGMutation());
814 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
815 DAG->addMutation(Mutation: createAMDGPUHazardLatencyDAGMutation(MF: C->MF));
816 return DAG;
817}
818
819static ScheduleDAGInstrs *
820createIterativeGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
821 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
822 auto *DAG = new GCNIterativeScheduler(
823 C, GCNIterativeScheduler::SCHEDULE_LEGACYMAXOCCUPANCY);
824 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
825 if (ST.shouldClusterStores())
826 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
827 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
828 return DAG;
829}
830
831static ScheduleDAGInstrs *createMinRegScheduler(MachineSchedContext *C) {
832 auto *DAG = new GCNIterativeScheduler(
833 C, GCNIterativeScheduler::SCHEDULE_MINREGFORCED);
834 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
835 return DAG;
836}
837
838static ScheduleDAGInstrs *
839createIterativeILPMachineScheduler(MachineSchedContext *C) {
840 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
841 auto *DAG = new GCNIterativeScheduler(C, GCNIterativeScheduler::SCHEDULE_ILP);
842 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
843 if (ST.shouldClusterStores())
844 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
845 DAG->addMutation(Mutation: createAMDGPUMacroFusionDAGMutation());
846 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
847 return DAG;
848}
849
850static MachineSchedRegistry
851SISchedRegistry("si", "Run SI's custom scheduler",
852 createSIMachineScheduler);
853
854static MachineSchedRegistry
855GCNMaxOccupancySchedRegistry("gcn-max-occupancy",
856 "Run GCN scheduler to maximize occupancy",
857 createGCNMaxOccupancyMachineScheduler);
858
859static MachineSchedRegistry
860 GCNMaxILPSchedRegistry("gcn-max-ilp", "Run GCN scheduler to maximize ilp",
861 createGCNMaxILPMachineScheduler);
862
863static MachineSchedRegistry GCNMaxMemoryClauseSchedRegistry(
864 "gcn-max-memory-clause", "Run GCN scheduler to maximize memory clause",
865 createGCNMaxMemoryClauseMachineScheduler);
866
867static MachineSchedRegistry IterativeGCNMaxOccupancySchedRegistry(
868 "gcn-iterative-max-occupancy-experimental",
869 "Run GCN scheduler to maximize occupancy (experimental)",
870 createIterativeGCNMaxOccupancyMachineScheduler);
871
872static MachineSchedRegistry GCNMinRegSchedRegistry(
873 "gcn-iterative-minreg",
874 "Run GCN iterative scheduler for minimal register usage (experimental)",
875 createMinRegScheduler);
876
877static MachineSchedRegistry GCNILPSchedRegistry(
878 "gcn-iterative-ilp",
879 "Run GCN iterative scheduler for ILP scheduling (experimental)",
880 createIterativeILPMachineScheduler);
881
882LLVM_READNONE
883static StringRef getGPUOrDefault(const Triple &TT, StringRef GPU) {
884 if (!GPU.empty())
885 return GPU;
886
887 if (StringRef Name = AMDGPU::getArchNameFromSubArch(SubArch: TT.getSubArch());
888 !Name.empty())
889 return Name;
890
891 // Need to default to a target with flat support for HSA.
892 if (TT.isAMDGCN())
893 return TT.getOS() == Triple::AMDHSA ? "generic-hsa" : "generic";
894
895 return "r600";
896}
897
898static Reloc::Model getEffectiveRelocModel() {
899 // The AMDGPU toolchain only supports generating shared objects, so we
900 // must always use PIC.
901 return Reloc::PIC_;
902}
903
904AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT,
905 StringRef CPU, StringRef FS,
906 const TargetOptions &Options,
907 std::optional<Reloc::Model> RM,
908 std::optional<CodeModel::Model> CM,
909 CodeGenOptLevel OptLevel)
910 : CodeGenTargetMachineImpl(
911 T, TT.computeDataLayout(), TT, getGPUOrDefault(TT, GPU: CPU), FS, Options,
912 getEffectiveRelocModel(), getEffectiveCodeModel(CM, Default: CodeModel::Small),
913 OptLevel),
914 TLOF(createTLOF(TT: getTargetTriple())) {
915 initAsmInfo();
916 if (TT.isAMDGCN()) {
917 // Triple is missing a representation for non-empty, but unrecognized
918 // subarches. Only permit no subarch for any subtarget if it was really
919 // empty.
920 bool IsUnknownSubArch =
921 TT.getSubArch() == Triple::NoSubArch && TT.getArchName().size() != 6;
922 if (IsUnknownSubArch)
923 reportFatalUsageError(reason: "unknown subarch " + TT.getArchName());
924
925 if (TT.getSubArch() != Triple::NoSubArch) {
926 AMDGPU::GPUKind Kind = AMDGPU::parseArchAMDGCN(CPU);
927 Triple::SubArchType GPUSubArch = AMDGPU::getSubArch(AK: Kind);
928 if (Kind != AMDGPU::GK_NONE && GPUSubArch != TT.getSubArch() &&
929 TT.getSubArch() != AMDGPU::getMajorSubArch(SubArch: GPUSubArch)) {
930 reportFatalUsageError(reason: "invalid cpu '" + CPU + "' for subarch " +
931 TT.getArchName());
932 }
933 }
934
935 if (getMCSubtargetInfo().checkFeatures(FS: "+wavefrontsize64"))
936 MRI.reset(p: llvm::createGCNMCRegisterInfo(DwarfFlavour: AMDGPUDwarfFlavour::Wave64));
937 else if (getMCSubtargetInfo().checkFeatures(FS: "+wavefrontsize32"))
938 MRI.reset(p: llvm::createGCNMCRegisterInfo(DwarfFlavour: AMDGPUDwarfFlavour::Wave32));
939 }
940 LLT::setUseExtended(true);
941}
942
943bool AMDGPUTargetMachine::EnableFunctionCalls = false;
944bool AMDGPUTargetMachine::EnableObjectLinking = false;
945bool AMDGPUTargetMachine::EnableLowerModuleLDS = true;
946
947AMDGPUTargetMachine::~AMDGPUTargetMachine() = default;
948
949StringRef AMDGPUTargetMachine::getGPUName(const Function &F) const {
950 Attribute GPUAttr = F.getFnAttribute(Kind: "target-cpu");
951 return GPUAttr.isValid() ? GPUAttr.getValueAsString() : getTargetCPU();
952}
953
954StringRef AMDGPUTargetMachine::getFeatureString(const Function &F) const {
955 Attribute FSAttr = F.getFnAttribute(Kind: "target-features");
956
957 return FSAttr.isValid() ? FSAttr.getValueAsString()
958 : getTargetFeatureString();
959}
960
961llvm::ScheduleDAGInstrs *
962AMDGPUTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
963 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
964 ScheduleDAGMILive *DAG = createSchedLive(C);
965 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
966 if (ST.shouldClusterStores())
967 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
968 return DAG;
969}
970
971/// Predicate for Internalize pass.
972static bool mustPreserveGV(const GlobalValue &GV) {
973 if (const Function *F = dyn_cast<Function>(Val: &GV))
974 return F->isDeclaration() || F->getName().starts_with(Prefix: "__asan_") ||
975 F->getName().starts_with(Prefix: "__sanitizer_") ||
976 AMDGPU::isEntryFunctionCC(CC: F->getCallingConv());
977
978 GV.removeDeadConstantUsers();
979 return !GV.use_empty();
980}
981
982void AMDGPUTargetMachine::registerDefaultAliasAnalyses(AAManager &AAM) {
983 if (EnableAMDGPUAliasAnalysis)
984 AAM.registerFunctionAnalysis<AMDGPUAA>();
985}
986
987static Expected<ScanOptions>
988parseAMDGPUAtomicOptimizerStrategy(StringRef Params) {
989 if (Params.empty())
990 return ScanOptions::Iterative;
991 Params.consume_front(Prefix: "strategy=");
992 auto Result = StringSwitch<std::optional<ScanOptions>>(Params)
993 .Case(S: "dpp", Value: ScanOptions::DPP)
994 .Cases(CaseStrings: {"iterative", ""}, Value: ScanOptions::Iterative)
995 .Case(S: "none", Value: ScanOptions::None)
996 .Default(Value: std::nullopt);
997 if (Result)
998 return *Result;
999 return make_error<StringError>(Args: "invalid parameter", Args: inconvertibleErrorCode());
1000}
1001
1002Expected<AMDGPUAttributorOptions>
1003parseAMDGPUAttributorPassOptions(StringRef Params) {
1004 AMDGPUAttributorOptions Result;
1005 while (!Params.empty()) {
1006 StringRef ParamName;
1007 std::tie(args&: ParamName, args&: Params) = Params.split(Separator: ';');
1008 if (ParamName == "closed-world") {
1009 Result.IsClosedWorld = true;
1010 } else {
1011 return make_error<StringError>(
1012 Args: formatv(Fmt: "invalid AMDGPUAttributor pass parameter '{0}' ", Vals&: ParamName)
1013 .str(),
1014 Args: inconvertibleErrorCode());
1015 }
1016 }
1017 return Result;
1018}
1019
1020void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
1021
1022#define GET_PASS_REGISTRY "AMDGPUPassRegistry.def"
1023#include "llvm/Passes/TargetPassRegistry.inc"
1024
1025 PB.registerPipelineParsingCallback(
1026 C: [this](StringRef Name, CGSCCPassManager &PM,
1027 ArrayRef<PassBuilder::PipelineElement> Pipeline) {
1028 if (Name == "amdgpu-attributor-cgscc" && getTargetTriple().isAMDGCN()) {
1029 PM.addPass(Pass: AMDGPUAttributorCGSCCPass(
1030 *static_cast<GCNTargetMachine *>(this)));
1031 return true;
1032 }
1033 return false;
1034 });
1035
1036 PB.registerScalarOptimizerLateEPCallback(
1037 C: [](FunctionPassManager &FPM, OptimizationLevel Level) {
1038 if (Level == OptimizationLevel::O0)
1039 return;
1040
1041 FPM.addPass(Pass: InferAddressSpacesPass());
1042 });
1043
1044 PB.registerVectorizerEndEPCallback(
1045 C: [](FunctionPassManager &FPM, OptimizationLevel Level) {
1046 if (Level == OptimizationLevel::O0)
1047 return;
1048
1049 FPM.addPass(Pass: InferAddressSpacesPass());
1050 });
1051
1052 PB.registerPipelineEarlySimplificationEPCallback(
1053 C: [this](ModulePassManager &PM, OptimizationLevel Level,
1054 ThinOrFullLTOPhase Phase) {
1055 if (!isLTOPreLink(Phase) && getTargetTriple().isAMDGCN()) {
1056 // When we are not using -fgpu-rdc, we can run accelerator code
1057 // selection relatively early, but still after linking to prevent
1058 // eager removal of potentially reachable symbols.
1059 if (EnableHipStdPar) {
1060 PM.addPass(Pass: HipStdParMathFixupPass());
1061 PM.addPass(Pass: HipStdParAcceleratorCodeSelectionPass());
1062 }
1063
1064 PM.addPass(Pass: AMDGPUPrintfRuntimeBindingPass());
1065 }
1066
1067 if (Level == OptimizationLevel::O0)
1068 return;
1069
1070 // We don't want to run internalization at per-module stage.
1071 if (InternalizeSymbols && !isLTOPreLink(Phase)) {
1072 PM.addPass(Pass: InternalizePass(mustPreserveGV));
1073 PM.addPass(Pass: GlobalDCEPass());
1074 }
1075
1076 if (EarlyInlineAll && !EnableFunctionCalls)
1077 PM.addPass(Pass: AMDGPUAlwaysInlinePass());
1078 });
1079
1080 PB.registerPeepholeEPCallback(
1081 C: [](FunctionPassManager &FPM, OptimizationLevel Level) {
1082 if (Level == OptimizationLevel::O0)
1083 return;
1084
1085 FPM.addPass(Pass: AMDGPUUseNativeCallsPass());
1086 if (EnableLibCallSimplify)
1087 FPM.addPass(Pass: AMDGPUSimplifyLibCallsPass());
1088
1089 if (EnableUniformIntrinsicCombine)
1090 FPM.addPass(Pass: AMDGPUUniformIntrinsicCombinePass());
1091 });
1092
1093 PB.registerCGSCCOptimizerLateEPCallback(
1094 C: [this](CGSCCPassManager &PM, OptimizationLevel Level) {
1095 if (Level == OptimizationLevel::O0)
1096 return;
1097
1098 FunctionPassManager FPM;
1099
1100 // Add promote kernel arguments pass to the opt pipeline right before
1101 // infer address spaces which is needed to do actual address space
1102 // rewriting.
1103 if (Level > OptimizationLevel::O1 && EnablePromoteKernelArguments)
1104 FPM.addPass(Pass: AMDGPUPromoteKernelArgumentsPass());
1105
1106 // Add infer address spaces pass to the opt pipeline after inlining
1107 // but before SROA to increase SROA opportunities.
1108 FPM.addPass(Pass: InferAddressSpacesPass());
1109
1110 // This should run after inlining to have any chance of doing
1111 // anything, and before other cleanup optimizations.
1112 FPM.addPass(Pass: AMDGPULowerKernelAttributesPass());
1113
1114 // Promote alloca to vector before SROA and loop unroll. If we
1115 // manage to eliminate allocas before unroll we may choose to unroll
1116 // less.
1117 FPM.addPass(Pass: AMDGPUPromoteAllocaToVectorPass(*this));
1118
1119 PM.addPass(Pass: createCGSCCToFunctionPassAdaptor(Pass: std::move(FPM)));
1120 });
1121
1122 // FIXME: Why is AMDGPUAttributor not in CGSCC?
1123 PB.registerOptimizerLastEPCallback(C: [this](ModulePassManager &MPM,
1124 OptimizationLevel Level,
1125 ThinOrFullLTOPhase Phase) {
1126 if (Level != OptimizationLevel::O0) {
1127 if (!isLTOPreLink(Phase)) {
1128 if (EnableAMDGPUAttributor && getTargetTriple().isAMDGCN()) {
1129 AMDGPUAttributorOptions Opts;
1130 MPM.addPass(Pass: AMDGPUAttributorPass(*this, Opts, Phase));
1131 }
1132 }
1133 }
1134 });
1135
1136 PB.registerFullLinkTimeOptimizationLastEPCallback(
1137 C: [this](ModulePassManager &PM, OptimizationLevel Level) {
1138 // Clean up redundant memory round-trips that the full-LTO pipeline,
1139 // unlike the non-LTO/ThinLTO ones, otherwise leaves for codegen.
1140 if (Level != OptimizationLevel::O0) {
1141 PM.addPass(Pass: createModuleToFunctionPassAdaptor(
1142 Pass: EarlyCSEPass(/*UseMemorySSA=*/true)));
1143 }
1144
1145 // When we are using -fgpu-rdc, we can only run accelerator code
1146 // selection after linking to prevent, otherwise we end up removing
1147 // potentially reachable symbols that were exported as external in other
1148 // modules.
1149 if (EnableHipStdPar) {
1150 PM.addPass(Pass: HipStdParMathFixupPass());
1151 PM.addPass(Pass: HipStdParAcceleratorCodeSelectionPass());
1152 }
1153 // We want to support the -lto-partitions=N option as "best effort".
1154 // For that, we need to lower LDS earlier in the pipeline before the
1155 // module is partitioned for codegen.
1156 if (EnableLowerExecSync)
1157 PM.addPass(Pass: AMDGPULowerExecSyncPass());
1158 if (EnableSwLowerLDS)
1159 PM.addPass(Pass: AMDGPUSwLowerLDSPass());
1160 if (EnableLowerModuleLDS)
1161 PM.addPass(Pass: AMDGPULowerModuleLDSPass(*this));
1162 if (Level != OptimizationLevel::O0) {
1163 // We only want to run this with O2 or higher since inliner and SROA
1164 // don't run in O1.
1165 if (Level != OptimizationLevel::O1) {
1166 PM.addPass(
1167 Pass: createModuleToFunctionPassAdaptor(Pass: InferAddressSpacesPass()));
1168 }
1169 // Do we really need internalization in LTO?
1170 if (InternalizeSymbols) {
1171 PM.addPass(Pass: InternalizePass(mustPreserveGV));
1172 PM.addPass(Pass: GlobalDCEPass());
1173 }
1174 if (EnableAMDGPUAttributor && getTargetTriple().isAMDGCN()) {
1175 AMDGPUAttributorOptions Opt;
1176 if (HasClosedWorldAssumption)
1177 Opt.IsClosedWorld = true;
1178 PM.addPass(Pass: AMDGPUAttributorPass(
1179 *this, Opt, ThinOrFullLTOPhase::FullLTOPostLink));
1180 }
1181 }
1182 if (!NoKernelInfoEndLTO) {
1183 FunctionPassManager FPM;
1184 FPM.addPass(Pass: KernelInfoPrinter(this));
1185 PM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
1186 }
1187 });
1188
1189 PB.registerRegClassFilterParsingCallback(
1190 C: [](StringRef FilterName) -> RegAllocFilterFunc {
1191 if (FilterName == "sgpr")
1192 return onlyAllocateSGPRs;
1193 if (FilterName == "vgpr")
1194 return onlyAllocateVGPRs;
1195 if (FilterName == "wwm")
1196 return onlyAllocateWWMRegs;
1197 return nullptr;
1198 });
1199}
1200
1201bool AMDGPUTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
1202 unsigned DestAS) const {
1203 return AMDGPU::isFlatGlobalAddrSpace(AS: SrcAS) &&
1204 AMDGPU::isFlatGlobalAddrSpace(AS: DestAS);
1205}
1206
1207unsigned AMDGPUTargetMachine::getAssumedAddrSpace(const Value *V) const {
1208 if (auto *Arg = dyn_cast<Argument>(Val: V);
1209 Arg &&
1210 AMDGPU::isModuleEntryFunctionCC(CC: Arg->getParent()->getCallingConv()) &&
1211 !Arg->hasByRefAttr())
1212 return AMDGPUAS::GLOBAL_ADDRESS;
1213
1214 const auto *LD = dyn_cast<LoadInst>(Val: V);
1215 if (!LD) // TODO: Handle invariant load like constant.
1216 return AMDGPUAS::UNKNOWN_ADDRESS_SPACE;
1217
1218 // It must be a generic pointer loaded.
1219 assert(V->getType()->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS);
1220
1221 const auto *Ptr = LD->getPointerOperand();
1222 if (Ptr->getType()->getPointerAddressSpace() != AMDGPUAS::CONSTANT_ADDRESS)
1223 return AMDGPUAS::UNKNOWN_ADDRESS_SPACE;
1224 // For a generic pointer loaded from the constant memory, it could be assumed
1225 // as a global pointer since the constant memory is only populated on the
1226 // host side. As implied by the offload programming model, only global
1227 // pointers could be referenced on the host side.
1228 return AMDGPUAS::GLOBAL_ADDRESS;
1229}
1230
1231std::pair<const Value *, unsigned>
1232AMDGPUTargetMachine::getPredicatedAddrSpace(const Value *V) const {
1233 if (auto *II = dyn_cast<IntrinsicInst>(Val: V)) {
1234 switch (II->getIntrinsicID()) {
1235 case Intrinsic::amdgcn_is_shared:
1236 return std::pair(II->getArgOperand(i: 0), AMDGPUAS::LOCAL_ADDRESS);
1237 case Intrinsic::amdgcn_is_private:
1238 return std::pair(II->getArgOperand(i: 0), AMDGPUAS::PRIVATE_ADDRESS);
1239 default:
1240 break;
1241 }
1242 return std::pair(nullptr, -1);
1243 }
1244 // Check the global pointer predication based on
1245 // (!is_share(p) && !is_private(p)). Note that logic 'and' is commutative and
1246 // the order of 'is_shared' and 'is_private' is not significant.
1247 Value *Ptr;
1248 if (match(
1249 V: const_cast<Value *>(V),
1250 P: m_c_And(L: m_Not(V: m_Intrinsic<Intrinsic::amdgcn_is_shared>(Ops: m_Value(V&: Ptr))),
1251 R: m_Not(V: m_Intrinsic<Intrinsic::amdgcn_is_private>(
1252 Ops: m_Deferred(V: Ptr))))))
1253 return std::pair(Ptr, AMDGPUAS::GLOBAL_ADDRESS);
1254
1255 return std::pair(nullptr, -1);
1256}
1257
1258unsigned
1259AMDGPUTargetMachine::getAddressSpaceForPseudoSourceKind(unsigned Kind) const {
1260 switch (Kind) {
1261 case PseudoSourceValue::Stack:
1262 case PseudoSourceValue::FixedStack:
1263 return AMDGPUAS::PRIVATE_ADDRESS;
1264 case PseudoSourceValue::ConstantPool:
1265 case PseudoSourceValue::GOT:
1266 case PseudoSourceValue::JumpTable:
1267 case PseudoSourceValue::GlobalValueCallEntry:
1268 case PseudoSourceValue::ExternalSymbolCallEntry:
1269 return AMDGPUAS::CONSTANT_ADDRESS;
1270 }
1271 return AMDGPUAS::FLAT_ADDRESS;
1272}
1273
1274bool AMDGPUTargetMachine::splitModule(
1275 Module &M, unsigned NumParts,
1276 function_ref<void(std::unique_ptr<Module> MPart)> ModuleCallback) {
1277 // FIXME(?): Would be better to use an already existing Analysis/PassManager,
1278 // but all current users of this API don't have one ready and would need to
1279 // create one anyway. Let's hide the boilerplate for now to keep it simple.
1280
1281 LoopAnalysisManager LAM;
1282 FunctionAnalysisManager FAM;
1283 CGSCCAnalysisManager CGAM;
1284 ModuleAnalysisManager MAM;
1285
1286 PassBuilder PB(this);
1287 PB.registerModuleAnalyses(MAM);
1288 PB.registerFunctionAnalyses(FAM);
1289 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
1290
1291 ModulePassManager MPM;
1292 MPM.addPass(Pass: AMDGPUSplitModulePass(NumParts, ModuleCallback));
1293 MPM.run(IR&: M, AM&: MAM);
1294 return true;
1295}
1296
1297//===----------------------------------------------------------------------===//
1298// GCN Target Machine (SI+)
1299//===----------------------------------------------------------------------===//
1300
1301GCNTargetMachine::GCNTargetMachine(const Target &T, const Triple &TT,
1302 StringRef CPU, StringRef FS,
1303 const TargetOptions &Options,
1304 std::optional<Reloc::Model> RM,
1305 std::optional<CodeModel::Model> CM,
1306 CodeGenOptLevel OL, bool JIT)
1307 : AMDGPUTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {
1308 setEnableDefaultMachineVerifier(false);
1309}
1310
1311enum class OOBFlagValue {
1312 Any = 0,
1313 Relaxed = 1,
1314 Strict = 2,
1315};
1316
1317/// Returns the OOB mode encoded by a module flag.
1318/// An absent flag defaults to Any.
1319static OOBFlagValue getOOBFlagValue(const Module &M, StringRef FlagName) {
1320 const auto *Flag =
1321 mdconst::dyn_extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: FlagName));
1322 if (!Flag)
1323 return OOBFlagValue::Any;
1324 return static_cast<OOBFlagValue>(Flag->getZExtValue());
1325}
1326
1327/// Returns the xnack/sramecc setting encoded by a module flag.
1328/// Module flag values: 0 = disabled, 1 = enabled.
1329/// An absent flag defaults to Any.
1330AMDGPU::TargetIDSetting
1331GCNTargetMachine::getTargetIDSettingFromModuleFlag(const Module &M,
1332 StringRef FlagName) {
1333 using AMDGPU::TargetIDSetting;
1334
1335 if (XnackSetting.getNumOccurrences() > 0 && FlagName == "amdgpu.xnack")
1336 return XnackSetting ? TargetIDSetting::On : TargetIDSetting::Off;
1337 if (SramEccSetting.getNumOccurrences() > 0 && FlagName == "amdgpu.sramecc")
1338 return SramEccSetting ? TargetIDSetting::On : TargetIDSetting::Off;
1339
1340 const auto *Flag =
1341 mdconst::dyn_extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: FlagName));
1342 if (!Flag)
1343 return TargetIDSetting::Any;
1344 return Flag->getZExtValue() == 0 ? TargetIDSetting::Off : TargetIDSetting::On;
1345}
1346
1347const TargetSubtargetInfo *
1348GCNTargetMachine::getSubtargetImpl(const Function &F) const {
1349 StringRef GPU = getGPUName(F);
1350 StringRef FS = getFeatureString(F);
1351
1352 const Module &M = *F.getParent();
1353 OOBFlagValue BufOOB = getOOBFlagValue(M, FlagName: AMDGPUOOBMode::BufferFlag);
1354 OOBFlagValue TBufOOB = getOOBFlagValue(M, FlagName: AMDGPUOOBMode::TBufferFlag);
1355 bool BufRelaxed = BufOOB == OOBFlagValue::Relaxed;
1356 bool TBufRelaxed = TBufOOB == OOBFlagValue::Relaxed;
1357
1358 using AMDGPU::TargetIDSetting;
1359 TargetIDSetting Xnack = getTargetIDSettingFromModuleFlag(M, FlagName: "amdgpu.xnack");
1360 TargetIDSetting SramEcc =
1361 getTargetIDSettingFromModuleFlag(M, FlagName: "amdgpu.sramecc");
1362
1363 SmallString<128> SubtargetKey(GPU);
1364 SubtargetKey.append(RHS: FS);
1365 if (BufRelaxed)
1366 SubtargetKey.append(RHS: ",buf-oob=1");
1367 if (TBufRelaxed)
1368 SubtargetKey.append(RHS: ",tbuf-oob=1");
1369 if (Xnack != TargetIDSetting::Any) {
1370 SubtargetKey.append(RHS: ",xnack=");
1371 SubtargetKey.push_back(Elt: Xnack == TargetIDSetting::On ? '1' : '0');
1372 }
1373 if (SramEcc != TargetIDSetting::Any) {
1374 SubtargetKey.append(RHS: ",sramecc=");
1375 SubtargetKey.push_back(Elt: Xnack == TargetIDSetting::On ? '1' : '0');
1376 }
1377
1378 auto &I = SubtargetMap[SubtargetKey];
1379 if (!I) {
1380 AMDGPU::GPUKind Kind = AMDGPU::parseArchAMDGCN(CPU: GPU);
1381 Triple::SubArchType GPUSubArch = AMDGPU::getSubArch(AK: Kind);
1382
1383 // Enforce the subtarget is covered by the subarch. Tolerate no subarch for
1384 // legacy compatibility.
1385 const Triple &TT = M.getTargetTriple();
1386 if (GPUSubArch != TT.getSubArch() && Kind != AMDGPU::GK_NONE) {
1387 // Check if this is a generic subarch which has subtargets. Ignore
1388 // unknown subtargets with a known subarch, since for whatever reason
1389 // the convention is to just print a warning and ignore unrecognized
1390 // subtargets.
1391 bool IsLegacyEmptySubArch = TT.getSubArch() == Triple::NoSubArch;
1392 if (!IsLegacyEmptySubArch &&
1393 AMDGPU::getMajorSubArch(SubArch: GPUSubArch) != TT.getSubArch()) {
1394 F.getContext().emitError(ErrorStr: "invalid subtarget '" + Twine(GPU) +
1395 "' for subarch " + TT.getArchName());
1396 }
1397 }
1398
1399 I = std::make_unique<GCNSubtarget>(args: TargetTriple, args&: GPU, args&: FS, args: *this, args&: BufRelaxed,
1400 args&: TBufRelaxed, args&: Xnack, args&: SramEcc);
1401 }
1402
1403 I->setScalarizeGlobalBehavior(ScalarizeGlobal);
1404
1405 return I.get();
1406}
1407
1408TargetTransformInfo
1409GCNTargetMachine::getTargetTransformInfo(const Function &F) const {
1410 return TargetTransformInfo(std::make_unique<GCNTTIImpl>(args: this, args: F));
1411}
1412
1413Error GCNTargetMachine::buildCodeGenPipeline(
1414 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
1415 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
1416 const CGPassBuilderOption &Opts, MCContext &Ctx,
1417 PassInstrumentationCallbacks *PIC) {
1418 AMDGPUCodeGenPassBuilder CGPB(*this, Opts, PIC);
1419 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
1420}
1421
1422ScheduleDAGInstrs *
1423GCNTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
1424 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
1425 if (ST.enableSIScheduler())
1426 return createSIMachineScheduler(C);
1427
1428 StringRef SchedStrategy = AMDGPU::getSchedStrategy(F: C->MF->getFunction());
1429
1430 if (SchedStrategy == "max-ilp")
1431 return createGCNMaxILPMachineScheduler(C);
1432
1433 if (SchedStrategy == "max-memory-clause")
1434 return createGCNMaxMemoryClauseMachineScheduler(C);
1435
1436 if (SchedStrategy == "iterative-ilp")
1437 return createIterativeILPMachineScheduler(C);
1438
1439 if (SchedStrategy == "iterative-minreg")
1440 return createMinRegScheduler(C);
1441
1442 if (SchedStrategy == "iterative-maxocc")
1443 return createIterativeGCNMaxOccupancyMachineScheduler(C);
1444
1445 if (SchedStrategy == "coexec") {
1446 diagnoseUnsupportedCoExecSchedulerSelection(F: C->MF->getFunction(), ST);
1447 return createGCNCoExecMachineScheduler(C);
1448 }
1449
1450 return createGCNMaxOccupancyMachineScheduler(C);
1451}
1452
1453ScheduleDAGInstrs *
1454GCNTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
1455 if (useNoopPostScheduler(F: C->MF->getFunction()))
1456 return createGCNNoopPostMachineScheduler(C);
1457
1458 ScheduleDAGMI *DAG =
1459 new GCNPostScheduleDAGMILive(C, std::make_unique<PostGenericScheduler>(args&: C),
1460 /*RemoveKillFlags=*/true);
1461 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
1462 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
1463 if (ST.shouldClusterStores())
1464 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII, TRI: DAG->TRI));
1465 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::PostRA));
1466 if ((EnableVOPD.getNumOccurrences() ||
1467 getOptLevel() >= CodeGenOptLevel::Less) &&
1468 EnableVOPD)
1469 DAG->addMutation(Mutation: createVOPDPairingMutation());
1470 DAG->addMutation(Mutation: createAMDGPUExportClusteringDAGMutation());
1471 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
1472 DAG->addMutation(Mutation: createAMDGPUHazardLatencyDAGMutation(MF: C->MF));
1473 return DAG;
1474}
1475//===----------------------------------------------------------------------===//
1476// AMDGPU Legacy Pass Setup
1477//===----------------------------------------------------------------------===//
1478
1479std::unique_ptr<CSEConfigBase> llvm::AMDGPUPassConfig::getCSEConfig() const {
1480 return getStandardCSEConfigForOpt(Level: TM->getOptLevel());
1481}
1482
1483namespace {
1484
1485class GCNPassConfig final : public AMDGPUPassConfig {
1486public:
1487 GCNPassConfig(TargetMachine &TM, PassManagerBase &PM)
1488 : AMDGPUPassConfig(TM, PM) {
1489 substitutePass(StandardID: &PostRASchedulerID, TargetID: &PostMachineSchedulerID);
1490 }
1491
1492 GCNTargetMachine &getGCNTargetMachine() const {
1493 return getTM<GCNTargetMachine>();
1494 }
1495
1496 bool addPreISel() override;
1497 void addMachineSSAOptimization() override;
1498 bool addILPOpts() override;
1499 bool addInstSelector() override;
1500 bool addIRTranslator() override;
1501 void addPreLegalizeMachineIR() override;
1502 bool addLegalizeMachineIR() override;
1503 void addPreRegBankSelect() override;
1504 bool addRegBankSelect() override;
1505 void addPreGlobalInstructionSelect() override;
1506 bool addGlobalInstructionSelect() override;
1507 void addPreRegAlloc() override;
1508 void addFastRegAlloc() override;
1509 void addOptimizedRegAlloc() override;
1510
1511 FunctionPass *createSGPRAllocPass(bool Optimized);
1512 FunctionPass *createVGPRAllocPass(bool Optimized);
1513 FunctionPass *createWWMRegAllocPass(bool Optimized);
1514 FunctionPass *createRegAllocPass(bool Optimized) override;
1515
1516 bool addRegAssignAndRewriteFast() override;
1517 bool addRegAssignAndRewriteOptimized() override;
1518
1519 bool addPreRewrite() override;
1520 void addPostRegAlloc() override;
1521 void addPreSched2() override;
1522 void addPreEmitPass() override;
1523 void addPostBBSections() override;
1524};
1525
1526} // end anonymous namespace
1527
1528AMDGPUPassConfig::AMDGPUPassConfig(TargetMachine &TM, PassManagerBase &PM)
1529 : TargetPassConfig(TM, PM) {
1530 // Exceptions and StackMaps are not supported, so these passes will never do
1531 // anything.
1532 disablePass(PassID: &StackMapLivenessID);
1533 disablePass(PassID: &FuncletLayoutID);
1534 // Garbage collection is not supported.
1535 disablePass(PassID: &GCLoweringID);
1536 disablePass(PassID: &ShadowStackGCLoweringID);
1537}
1538
1539void AMDGPUPassConfig::addEarlyCSEOrGVNPass() {
1540 if (getOptLevel() == CodeGenOptLevel::Aggressive)
1541 addPass(P: createGVNPass());
1542 else
1543 addPass(P: createEarlyCSEPass());
1544}
1545
1546void AMDGPUPassConfig::addStraightLineScalarOptimizationPasses() {
1547 if (isPassEnabled(Opt: EnableLoopPrefetch, Level: CodeGenOptLevel::Aggressive))
1548 addPass(P: createLoopDataPrefetchPass());
1549 addPass(P: createSeparateConstOffsetFromGEPPass());
1550 // ReassociateGEPs exposes more opportunities for SLSR. See
1551 // the example in reassociate-geps-and-slsr.ll.
1552 addPass(P: createStraightLineStrengthReducePass());
1553 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or
1554 // EarlyCSE can reuse.
1555 addEarlyCSEOrGVNPass();
1556 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
1557 addPass(P: createNaryReassociatePass());
1558 // NaryReassociate on GEPs creates redundant common expressions, so run
1559 // EarlyCSE after it.
1560 addPass(P: createEarlyCSEPass());
1561}
1562
1563void AMDGPUPassConfig::addIRPasses() {
1564 const AMDGPUTargetMachine &TM = getAMDGPUTargetMachine();
1565
1566 if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN())
1567 addPass(P: createAMDGPURemoveIncompatibleFunctionsPass(&TM));
1568
1569 // There is no reason to run these.
1570 disablePass(PassID: &StackMapLivenessID);
1571 disablePass(PassID: &FuncletLayoutID);
1572 disablePass(PassID: &PatchableFunctionID);
1573
1574 if (TM.getTargetTriple().isAMDGCN())
1575 addPass(P: createAMDGPUPrintfRuntimeBinding());
1576
1577 if (LowerCtorDtor)
1578 addPass(P: createAMDGPUCtorDtorLoweringLegacyPass());
1579
1580 if (TM.getTargetTriple().isAMDGCN() &&
1581 isPassEnabled(Opt: EnableImageIntrinsicOptimizer))
1582 addPass(P: createAMDGPUImageIntrinsicOptimizerPass(&TM));
1583
1584 if (EnableUniformIntrinsicCombine)
1585 addPass(P: createAMDGPUUniformIntrinsicCombineLegacyPass());
1586
1587 // This can be disabled by passing ::Disable here or on the command line
1588 // with --expand-variadics-override=disable.
1589 addPass(P: createExpandVariadicsPass(ExpandVariadicsMode::Lowering));
1590
1591 // Function calls are not supported, so make sure we inline everything.
1592 addPass(P: createAMDGPUAlwaysInlinePass());
1593 addPass(P: createAlwaysInlinerLegacyPass());
1594
1595 // Handle uses of OpenCL image2d_t, image3d_t and sampler_t arguments.
1596 if (TM.getTargetTriple().getArch() == Triple::r600)
1597 addPass(P: createR600OpenCLImageTypeLoweringPass());
1598
1599 // Make enqueued block runtime handles externally visible.
1600 addPass(P: createAMDGPUExportKernelRuntimeHandlesLegacyPass());
1601
1602 // Lower special LDS accesses.
1603 if (EnableLowerExecSync)
1604 addPass(P: createAMDGPULowerExecSyncLegacyPass());
1605
1606 // Lower LDS accesses to global memory pass if address sanitizer is enabled.
1607 if (EnableSwLowerLDS)
1608 addPass(P: createAMDGPUSwLowerLDSLegacyPass());
1609
1610 // Runs before PromoteAlloca so the latter can account for function uses
1611 if (EnableLowerModuleLDS) {
1612 addPass(P: createAMDGPULowerModuleLDSLegacyPass(TM: &TM));
1613 }
1614
1615 // Run atomic optimizer before Atomic Expand
1616 if ((TM.getTargetTriple().isAMDGCN()) &&
1617 (TM.getOptLevel() >= CodeGenOptLevel::Less) &&
1618 (AMDGPUAtomicOptimizerStrategy != ScanOptions::None)) {
1619 addPass(P: createAMDGPUAtomicOptimizerPass(ScanStrategy: AMDGPUAtomicOptimizerStrategy));
1620 }
1621
1622 addPass(P: createAtomicExpandLegacyPass());
1623
1624 if (TM.getOptLevel() > CodeGenOptLevel::None) {
1625 addPass(P: createAMDGPUPromoteAlloca());
1626
1627 if (isPassEnabled(Opt: EnableScalarIRPasses))
1628 addStraightLineScalarOptimizationPasses();
1629
1630 if (EnableAMDGPUAliasAnalysis) {
1631 addPass(P: createAMDGPUAAWrapperPass());
1632 addPass(P: createExternalAAWrapperPass(Callback: [](Pass &P, Function &,
1633 AAResults &AAR) {
1634 if (auto *WrapperPass = P.getAnalysisIfAvailable<AMDGPUAAWrapperPass>())
1635 AAR.addAAResult(AAResult&: WrapperPass->getResult());
1636 }));
1637 }
1638
1639 if (TM.getTargetTriple().isAMDGCN()) {
1640 // TODO: May want to move later or split into an early and late one.
1641 addPass(P: createAMDGPUCodeGenPreparePass());
1642 }
1643
1644 // Try to hoist loop invariant parts of divisions AMDGPUCodeGenPrepare may
1645 // have expanded.
1646 if (TM.getOptLevel() > CodeGenOptLevel::Less)
1647 addPass(P: createLICMPass());
1648 }
1649
1650 TargetPassConfig::addIRPasses();
1651
1652 // EarlyCSE is not always strong enough to clean up what LSR produces. For
1653 // example, GVN can combine
1654 //
1655 // %0 = add %a, %b
1656 // %1 = add %b, %a
1657 //
1658 // and
1659 //
1660 // %0 = shl nsw %a, 2
1661 // %1 = shl %a, 2
1662 //
1663 // but EarlyCSE can do neither of them.
1664 if (isPassEnabled(Opt: EnableScalarIRPasses))
1665 addEarlyCSEOrGVNPass();
1666}
1667
1668void AMDGPUPassConfig::addCodeGenPrepare() {
1669 if (TM->getTargetTriple().isAMDGCN() &&
1670 TM->getOptLevel() > CodeGenOptLevel::None)
1671 addPass(P: createAMDGPUPreloadKernelArgumentsLegacyPass(TM));
1672
1673 if (TM->getTargetTriple().isAMDGCN() && EnableLowerKernelArguments)
1674 addPass(P: createAMDGPULowerKernelArgumentsPass());
1675
1676 TargetPassConfig::addCodeGenPrepare();
1677
1678 if (isPassEnabled(Opt: EnableLoadStoreVectorizer))
1679 addPass(P: createLoadStoreVectorizerPass());
1680
1681 if (TM->getTargetTriple().isAMDGCN()) {
1682 // This lowering has been placed after codegenprepare to take advantage of
1683 // address mode matching (which is why it isn't put with the LDS lowerings).
1684 // It could be placed anywhere before uniformity annotations (an analysis
1685 // that it changes by splitting up fat pointers into their components)
1686 // but has been put before switch lowering and CFG flattening so that those
1687 // passes can run on the more optimized control flow this pass creates in
1688 // many cases.
1689 addPass(P: createAMDGPULowerBufferFatPointersPass());
1690 addPass(P: createAMDGPULowerIntrinsicsLegacyPass());
1691 }
1692
1693 // LowerSwitch pass may introduce unreachable blocks that can
1694 // cause unexpected behavior for subsequent passes. Placing it
1695 // here seems better that these blocks would get cleaned up by
1696 // UnreachableBlockElim inserted next in the pass flow.
1697 addPass(P: createLowerSwitchPass());
1698}
1699
1700bool AMDGPUPassConfig::addPreISel() {
1701 if (TM->getOptLevel() > CodeGenOptLevel::None)
1702 addPass(P: createFlattenCFGPass());
1703 return false;
1704}
1705
1706bool AMDGPUPassConfig::addInstSelector() {
1707 addPass(P: createAMDGPUISelDag(TM&: getAMDGPUTargetMachine(), OptLevel: getOptLevel()));
1708 return false;
1709}
1710
1711bool AMDGPUPassConfig::addGCPasses() {
1712 // Do nothing. GC is not supported.
1713 return false;
1714}
1715
1716//===----------------------------------------------------------------------===//
1717// GCN Legacy Pass Setup
1718//===----------------------------------------------------------------------===//
1719
1720bool GCNPassConfig::addPreISel() {
1721 AMDGPUPassConfig::addPreISel();
1722
1723 if (TM->getOptLevel() > CodeGenOptLevel::None) {
1724 addPass(P: createSinkingPass());
1725 addPass(P: createAMDGPULateCodeGenPrepareLegacyPass());
1726 }
1727
1728 // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
1729 // regions formed by them.
1730 addPass(PassID: &AMDGPUUnifyDivergentExitNodesID);
1731 addPass(P: createFixIrreduciblePass());
1732 addPass(P: createUnifyLoopExitsPass());
1733 addPass(P: createStructurizeCFGPass(SkipUniformRegions: false)); // true -> SkipUniformRegions
1734
1735 addPass(P: createAMDGPUAnnotateUniformValuesLegacy());
1736 addPass(P: createSIAnnotateControlFlowLegacyPass());
1737 // TODO: Move this right after structurizeCFG to avoid extra divergence
1738 // analysis. This depends on stopping SIAnnotateControlFlow from making
1739 // control flow modifications.
1740 addPass(P: createAMDGPURewriteUndefForPHILegacyPass());
1741
1742 // SDAG requires LCSSA, GlobalISel does not. Disable LCSSA for -global-isel
1743 // without any of the fallback options.
1744 if (getCGPassBuilderOption().EnableGlobalISelOption !=
1745 cl::boolOrDefault::BOU_TRUE ||
1746 !isGlobalISelAbortEnabled())
1747 addPass(P: createLCSSAPass());
1748
1749 if (TM->getOptLevel() > CodeGenOptLevel::Less)
1750 addPass(PassID: &AMDGPUPerfHintAnalysisLegacyID);
1751
1752 return false;
1753}
1754
1755void GCNPassConfig::addMachineSSAOptimization() {
1756 TargetPassConfig::addMachineSSAOptimization();
1757
1758 // We want to fold operands after PeepholeOptimizer has run (or as part of
1759 // it), because it will eliminate extra copies making it easier to fold the
1760 // real source operand. We want to eliminate dead instructions after, so that
1761 // we see fewer uses of the copies. We then need to clean up the dead
1762 // instructions leftover after the operands are folded as well.
1763 //
1764 // XXX - Can we get away without running DeadMachineInstructionElim again?
1765 addPass(PassID: &SIFoldOperandsLegacyID);
1766 if (EnableDPPCombine)
1767 addPass(PassID: &GCNDPPCombineLegacyID);
1768 addPass(PassID: &SILoadStoreOptimizerLegacyID);
1769 if (isPassEnabled(Opt: EnableSDWAPeephole)) {
1770 addPass(PassID: &SIPeepholeSDWALegacyID);
1771 addPass(PassID: &EarlyMachineLICMID);
1772 addPass(PassID: &MachineCSELegacyID);
1773 addPass(PassID: &SIFoldOperandsLegacyID);
1774 }
1775 addPass(PassID: &DeadMachineInstructionElimID);
1776 addPass(P: createSIShrinkInstructionsLegacyPass());
1777}
1778
1779bool GCNPassConfig::addILPOpts() {
1780 if (EnableEarlyIfConversion)
1781 addPass(PassID: &EarlyIfConverterLegacyID);
1782
1783 TargetPassConfig::addILPOpts();
1784 return false;
1785}
1786
1787bool GCNPassConfig::addInstSelector() {
1788 AMDGPUPassConfig::addInstSelector();
1789 addPass(PassID: &SIFixSGPRCopiesLegacyID);
1790 addPass(P: createSILowerI1CopiesLegacyPass());
1791 return false;
1792}
1793
1794bool GCNPassConfig::addIRTranslator() {
1795 addPass(P: new IRTranslatorLegacy(getOptLevel()));
1796 return false;
1797}
1798
1799void GCNPassConfig::addPreLegalizeMachineIR() {
1800 bool IsOptLevelNone = getOptLevel() == CodeGenOptLevel::None;
1801 addPass(P: createAMDGPUPreLegalizeCombinerLegacyPass(IsOptLevelNone));
1802 addPass(P: new LocalizerLegacy());
1803}
1804
1805bool GCNPassConfig::addLegalizeMachineIR() {
1806 addPass(P: new LegalizerLegacy());
1807 return false;
1808}
1809
1810void GCNPassConfig::addPreRegBankSelect() {
1811 bool IsOptNone = getOptLevel() == CodeGenOptLevel::None;
1812 addPass(P: createAMDGPUPostLegalizeCombinerLegacy(IsOptNone));
1813 addPass(P: createAMDGPUGlobalISelDivergenceLoweringPass());
1814}
1815
1816bool GCNPassConfig::addRegBankSelect() {
1817 addPass(P: createAMDGPURegBankSelectLegacyPass());
1818 addPass(P: createAMDGPURegBankLegalizeLegacyPass());
1819 return false;
1820}
1821
1822void GCNPassConfig::addPreGlobalInstructionSelect() {
1823 bool IsOptLevelNone = getOptLevel() == CodeGenOptLevel::None;
1824 addPass(P: createAMDGPURegBankCombinerLegacy(IsOptLevelNone));
1825}
1826
1827bool GCNPassConfig::addGlobalInstructionSelect() {
1828 addPass(P: new InstructionSelectLegacy(getOptLevel()));
1829 return false;
1830}
1831
1832void GCNPassConfig::addFastRegAlloc() {
1833 // FIXME: We have to disable the verifier here because of PHIElimination +
1834 // TwoAddressInstructions disabling it.
1835
1836 // This must be run immediately after phi elimination and before
1837 // TwoAddressInstructions, otherwise the processing of the tied operand of
1838 // SI_ELSE will introduce a copy of the tied operand source after the else.
1839 insertPass(TargetPassID: &PHIEliminationID, InsertedPassID: &SILowerControlFlowLegacyID);
1840
1841 insertPass(TargetPassID: &TwoAddressInstructionPassID, InsertedPassID: &SIWholeQuadModeID);
1842
1843 TargetPassConfig::addFastRegAlloc();
1844}
1845
1846void GCNPassConfig::addPreRegAlloc() {
1847 if (getOptLevel() != CodeGenOptLevel::None)
1848 addPass(PassID: &AMDGPUPrepareAGPRAllocLegacyID);
1849 if (getOptLevel() >= CodeGenOptLevel::Default && EnableMachinePipeliner)
1850 addPass(PassID: &MachinePipelinerID);
1851}
1852
1853void GCNPassConfig::addOptimizedRegAlloc() {
1854 if (EnableDCEInRA)
1855 insertPass(TargetPassID: &DetectDeadLanesID, InsertedPassID: &DeadMachineInstructionElimID);
1856
1857 // FIXME: when an instruction has a Killed operand, and the instruction is
1858 // inside a bundle, seems only the BUNDLE instruction appears as the Kills of
1859 // the register in LiveVariables, this would trigger a failure in verifier,
1860 // we should fix it and enable the verifier.
1861 if (OptVGPRLiveRange)
1862 insertPass(TargetPassID: &LiveVariablesID, InsertedPassID: &SIOptimizeVGPRLiveRangeLegacyID);
1863
1864 // This must be run immediately after phi elimination and before
1865 // TwoAddressInstructions, otherwise the processing of the tied operand of
1866 // SI_ELSE will introduce a copy of the tied operand source after the else.
1867 insertPass(TargetPassID: &PHIEliminationID, InsertedPassID: &SILowerControlFlowLegacyID);
1868
1869 if (EnableRewritePartialRegUses)
1870 insertPass(TargetPassID: &RenameIndependentSubregsID, InsertedPassID: &GCNRewritePartialRegUsesID);
1871
1872 if (isPassEnabled(Opt: EnablePreRAOptimizations))
1873 insertPass(TargetPassID: &MachineSchedulerID, InsertedPassID: &GCNPreRAOptimizationsID);
1874
1875 // Allow the scheduler to run before SIWholeQuadMode inserts exec manipulation
1876 // instructions that cause scheduling barriers.
1877 insertPass(TargetPassID: &MachineSchedulerID, InsertedPassID: &SIWholeQuadModeID);
1878
1879 if (OptExecMaskPreRA)
1880 insertPass(TargetPassID: &MachineSchedulerID, InsertedPassID: &SIOptimizeExecMaskingPreRAID);
1881
1882 // This is not an essential optimization and it has a noticeable impact on
1883 // compilation time, so we only enable it from O2.
1884 if (TM->getOptLevel() > CodeGenOptLevel::Less)
1885 insertPass(TargetPassID: &MachineSchedulerID, InsertedPassID: &SIFormMemoryClausesID);
1886
1887 TargetPassConfig::addOptimizedRegAlloc();
1888}
1889
1890bool GCNPassConfig::addPreRewrite() {
1891 if (EnableRegReassign)
1892 addPass(PassID: &GCNNSAReassignID);
1893
1894 addPass(PassID: &AMDGPURewriteAGPRCopyMFMALegacyID);
1895 return true;
1896}
1897
1898FunctionPass *GCNPassConfig::createSGPRAllocPass(bool Optimized) {
1899 // Initialize the global default.
1900 llvm::call_once(flag&: InitializeDefaultSGPRRegisterAllocatorFlag,
1901 F&: initializeDefaultSGPRRegisterAllocatorOnce);
1902
1903 RegisterRegAlloc::FunctionPassCtor Ctor = SGPRRegisterRegAlloc::getDefault();
1904 if (Ctor != useDefaultRegisterAllocator)
1905 return Ctor();
1906
1907 if (Optimized)
1908 return createGreedyRegisterAllocator(F: onlyAllocateSGPRs);
1909
1910 return createFastRegisterAllocator(F: onlyAllocateSGPRs, ClearVirtRegs: false);
1911}
1912
1913FunctionPass *GCNPassConfig::createVGPRAllocPass(bool Optimized) {
1914 // Initialize the global default.
1915 llvm::call_once(flag&: InitializeDefaultVGPRRegisterAllocatorFlag,
1916 F&: initializeDefaultVGPRRegisterAllocatorOnce);
1917
1918 RegisterRegAlloc::FunctionPassCtor Ctor = VGPRRegisterRegAlloc::getDefault();
1919 if (Ctor != useDefaultRegisterAllocator)
1920 return Ctor();
1921
1922 if (Optimized)
1923 return createGreedyVGPRRegisterAllocator();
1924
1925 return createFastVGPRRegisterAllocator();
1926}
1927
1928FunctionPass *GCNPassConfig::createWWMRegAllocPass(bool Optimized) {
1929 // Initialize the global default.
1930 llvm::call_once(flag&: InitializeDefaultWWMRegisterAllocatorFlag,
1931 F&: initializeDefaultWWMRegisterAllocatorOnce);
1932
1933 RegisterRegAlloc::FunctionPassCtor Ctor = WWMRegisterRegAlloc::getDefault();
1934 if (Ctor != useDefaultRegisterAllocator)
1935 return Ctor();
1936
1937 if (Optimized)
1938 return createGreedyWWMRegisterAllocator();
1939
1940 return createFastWWMRegisterAllocator();
1941}
1942
1943FunctionPass *GCNPassConfig::createRegAllocPass(bool Optimized) {
1944 llvm_unreachable("should not be used");
1945}
1946
1947static const char RegAllocOptNotSupportedMessage[] =
1948 "-regalloc not supported with amdgcn. Use -sgpr-regalloc, -wwm-regalloc, "
1949 "and -vgpr-regalloc";
1950
1951bool GCNPassConfig::addRegAssignAndRewriteFast() {
1952 if (!usingDefaultRegAlloc())
1953 reportFatalUsageError(reason: RegAllocOptNotSupportedMessage);
1954
1955 addPass(PassID: &GCNPreRALongBranchRegID);
1956
1957 addPass(P: createSGPRAllocPass(Optimized: false));
1958
1959 // Equivalent of PEI for SGPRs.
1960 addPass(PassID: &SILowerSGPRSpillsLegacyID);
1961
1962 // To Allocate wwm registers used in whole quad mode operations (for shaders).
1963 addPass(PassID: &SIPreAllocateWWMRegsLegacyID);
1964
1965 // For allocating other wwm register operands.
1966 addPass(P: createWWMRegAllocPass(Optimized: false));
1967
1968 addPass(PassID: &SILowerWWMCopiesLegacyID);
1969 addPass(PassID: &AMDGPUReserveWWMRegsLegacyID);
1970
1971 // For allocating per-thread VGPRs.
1972 addPass(P: createVGPRAllocPass(Optimized: false));
1973
1974 return true;
1975}
1976
1977bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
1978 if (!usingDefaultRegAlloc())
1979 reportFatalUsageError(reason: RegAllocOptNotSupportedMessage);
1980
1981 addPass(PassID: &GCNPreRALongBranchRegID);
1982
1983 addPass(P: createSGPRAllocPass(Optimized: true));
1984
1985 // Commit allocated register changes. This is mostly necessary because too
1986 // many things rely on the use lists of the physical registers, such as the
1987 // verifier. This is only necessary with allocators which use LiveIntervals,
1988 // since FastRegAlloc does the replacements itself.
1989 addPass(P: createVirtRegRewriter(ClearVirtRegs: false));
1990
1991 // At this point, the sgpr-regalloc has been done and it is good to have the
1992 // stack slot coloring to try to optimize the SGPR spill stack indices before
1993 // attempting the custom SGPR spill lowering.
1994 addPass(PassID: &StackSlotColoringID);
1995
1996 // Equivalent of PEI for SGPRs.
1997 addPass(PassID: &SILowerSGPRSpillsLegacyID);
1998
1999 // To Allocate wwm registers used in whole quad mode operations (for shaders).
2000 addPass(PassID: &SIPreAllocateWWMRegsLegacyID);
2001
2002 // For allocating other whole wave mode registers.
2003 addPass(P: createWWMRegAllocPass(Optimized: true));
2004 addPass(PassID: &SILowerWWMCopiesLegacyID);
2005 addPass(P: createVirtRegRewriter(ClearVirtRegs: false));
2006 addPass(PassID: &AMDGPUReserveWWMRegsLegacyID);
2007
2008 // For allocating per-thread VGPRs.
2009 addPass(P: createVGPRAllocPass(Optimized: true));
2010
2011 addPreRewrite();
2012 addPass(PassID: &VirtRegRewriterID);
2013
2014 addPass(PassID: &AMDGPUMarkLastScratchLoadID);
2015
2016 return true;
2017}
2018
2019void GCNPassConfig::addPostRegAlloc() {
2020 addPass(PassID: &SIFixVGPRCopiesID);
2021 if (getOptLevel() > CodeGenOptLevel::None)
2022 addPass(PassID: &SIOptimizeExecMaskingLegacyID);
2023 TargetPassConfig::addPostRegAlloc();
2024}
2025
2026void GCNPassConfig::addPreSched2() {
2027 if (TM->getOptLevel() > CodeGenOptLevel::None)
2028 addPass(P: createSIShrinkInstructionsLegacyPass());
2029 addPass(PassID: &SIPostRABundlerLegacyID);
2030}
2031
2032void GCNPassConfig::addPreEmitPass() {
2033 if (isPassEnabled(Opt: EnableVOPD, Level: CodeGenOptLevel::Less))
2034 addPass(PassID: &GCNCreateVOPDID);
2035 addPass(P: createSIMemoryLegalizerPass());
2036 addPass(P: createSIInsertWaitcntsPass());
2037
2038 addPass(P: createSIModeRegisterPass());
2039
2040 if (getOptLevel() > CodeGenOptLevel::None)
2041 addPass(PassID: &SIInsertHardClausesID);
2042
2043 addPass(PassID: &SILateBranchLoweringPassID);
2044 if (isPassEnabled(Opt: EnableSetWavePriority, Level: CodeGenOptLevel::Less))
2045 addPass(P: createAMDGPUSetWavePriorityPass());
2046 if (getOptLevel() > CodeGenOptLevel::None)
2047 addPass(PassID: &SIPreEmitPeepholeID);
2048 // The hazard recognizer that runs as part of the post-ra scheduler does not
2049 // guarantee to be able handle all hazards correctly. This is because if there
2050 // are multiple scheduling regions in a basic block, the regions are scheduled
2051 // bottom up, so when we begin to schedule a region we don't know what
2052 // instructions were emitted directly before it.
2053 //
2054 // Here we add a stand-alone hazard recognizer pass which can handle all
2055 // cases.
2056 addPass(PassID: &PostRAHazardRecognizerID);
2057
2058 addPass(PassID: &AMDGPUWaitSGPRHazardsLegacyID);
2059
2060 addPass(PassID: &AMDGPULowerVGPREncodingLegacyID);
2061
2062 if (isPassEnabled(Opt: EnableInsertDelayAlu, Level: CodeGenOptLevel::Less))
2063 addPass(PassID: &AMDGPUInsertDelayAluID);
2064
2065 addPass(PassID: &BranchRelaxationPassID);
2066}
2067
2068void GCNPassConfig::addPostBBSections() {
2069 // We run this later to avoid passes like livedebugvalues and BBSections
2070 // having to deal with the apparent multi-entry functions we may generate.
2071 addPass(P: createAMDGPUPreloadKernArgPrologLegacyPass());
2072}
2073
2074TargetPassConfig *GCNTargetMachine::createPassConfig(PassManagerBase &PM) {
2075 return new GCNPassConfig(*this, PM);
2076}
2077
2078void GCNTargetMachine::registerMachineRegisterInfoCallback(
2079 MachineFunction &MF) const {
2080 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2081 MF.getRegInfo().addDelegate(delegate: MFI);
2082}
2083
2084MachineFunctionInfo *GCNTargetMachine::createMachineFunctionInfo(
2085 BumpPtrAllocator &Allocator, const Function &F,
2086 const TargetSubtargetInfo *STI) const {
2087 return SIMachineFunctionInfo::create<SIMachineFunctionInfo>(
2088 Allocator, F, STI: static_cast<const GCNSubtarget *>(STI));
2089}
2090
2091yaml::MachineFunctionInfo *GCNTargetMachine::createDefaultFuncInfoYAML() const {
2092 return new yaml::SIMachineFunctionInfo();
2093}
2094
2095yaml::MachineFunctionInfo *
2096GCNTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
2097 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2098 return new yaml::SIMachineFunctionInfo(
2099 *MFI, *MF.getSubtarget<GCNSubtarget>().getRegisterInfo(), MF);
2100}
2101
2102bool GCNTargetMachine::parseMachineFunctionInfo(
2103 const yaml::MachineFunctionInfo &MFI_, PerFunctionMIParsingState &PFS,
2104 SMDiagnostic &Error, SMRange &SourceRange) const {
2105 const yaml::SIMachineFunctionInfo &YamlMFI =
2106 static_cast<const yaml::SIMachineFunctionInfo &>(MFI_);
2107 MachineFunction &MF = PFS.MF;
2108 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2109 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2110
2111 if (MFI->initializeBaseYamlFields(YamlMFI, MF, PFS, Error, SourceRange))
2112 return true;
2113
2114 if (MFI->Occupancy == 0) {
2115 // Fixup the subtarget dependent default value.
2116 MFI->Occupancy = ST.getOccupancyWithWorkGroupSizes(MF).second;
2117 }
2118
2119 auto parseRegister = [&](const yaml::StringValue &RegName, Register &RegVal) {
2120 Register TempReg;
2121 if (parseNamedRegisterReference(PFS, Reg&: TempReg, Src: RegName.Value, Error)) {
2122 SourceRange = RegName.SourceRange;
2123 return true;
2124 }
2125 RegVal = TempReg;
2126
2127 return false;
2128 };
2129
2130 auto parseOptionalRegister = [&](const yaml::StringValue &RegName,
2131 Register &RegVal) {
2132 return !RegName.Value.empty() && parseRegister(RegName, RegVal);
2133 };
2134
2135 if (parseOptionalRegister(YamlMFI.VGPRForAGPRCopy, MFI->VGPRForAGPRCopy))
2136 return true;
2137
2138 if (parseOptionalRegister(YamlMFI.SGPRForEXECCopy, MFI->SGPRForEXECCopy))
2139 return true;
2140
2141 if (parseOptionalRegister(YamlMFI.LongBranchReservedReg,
2142 MFI->LongBranchReservedReg))
2143 return true;
2144
2145 auto diagnoseRegisterClass = [&](const yaml::StringValue &RegName) {
2146 // Create a diagnostic for a the register string literal.
2147 const MemoryBuffer &Buffer =
2148 *PFS.SM->getMemoryBuffer(i: PFS.SM->getMainFileID());
2149 Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1,
2150 RegName.Value.size(), SourceMgr::DK_Error,
2151 "incorrect register class for field", RegName.Value,
2152 {}, {});
2153 SourceRange = RegName.SourceRange;
2154 return true;
2155 };
2156
2157 if (parseRegister(YamlMFI.ScratchRSrcReg, MFI->ScratchRSrcReg) ||
2158 parseRegister(YamlMFI.FrameOffsetReg, MFI->FrameOffsetReg) ||
2159 parseRegister(YamlMFI.StackPtrOffsetReg, MFI->StackPtrOffsetReg))
2160 return true;
2161
2162 if (MFI->ScratchRSrcReg != AMDGPU::PRIVATE_RSRC_REG &&
2163 !AMDGPU::SGPR_128RegClass.contains(Reg: MFI->ScratchRSrcReg)) {
2164 return diagnoseRegisterClass(YamlMFI.ScratchRSrcReg);
2165 }
2166
2167 if (MFI->FrameOffsetReg != AMDGPU::FP_REG &&
2168 !AMDGPU::SGPR_32RegClass.contains(Reg: MFI->FrameOffsetReg)) {
2169 return diagnoseRegisterClass(YamlMFI.FrameOffsetReg);
2170 }
2171
2172 if (MFI->StackPtrOffsetReg != AMDGPU::SP_REG &&
2173 !AMDGPU::SGPR_32RegClass.contains(Reg: MFI->StackPtrOffsetReg)) {
2174 return diagnoseRegisterClass(YamlMFI.StackPtrOffsetReg);
2175 }
2176
2177 for (const auto &YamlReg : YamlMFI.WWMReservedRegs) {
2178 Register ParsedReg;
2179 if (parseRegister(YamlReg, ParsedReg))
2180 return true;
2181
2182 MFI->reserveWWMRegister(Reg: ParsedReg);
2183 }
2184
2185 for (const auto &[_, Info] : PFS.VRegInfosNamed) {
2186 MFI->setFlag(Reg: Info->VReg, Flag: Info->Flags);
2187 }
2188 for (const auto &[_, Info] : PFS.VRegInfos) {
2189 MFI->setFlag(Reg: Info->VReg, Flag: Info->Flags);
2190 }
2191
2192 for (const auto &YamlRegStr : YamlMFI.SpillPhysVGPRS) {
2193 Register ParsedReg;
2194 if (parseRegister(YamlRegStr, ParsedReg))
2195 return true;
2196 MFI->SpillPhysVGPRs.push_back(Elt: ParsedReg);
2197 }
2198
2199 auto parseAndCheckArgument = [&](const std::optional<yaml::SIArgument> &A,
2200 const TargetRegisterClass &RC,
2201 ArgDescriptor &Arg, unsigned UserSGPRs,
2202 unsigned SystemSGPRs) {
2203 // Skip parsing if it's not present.
2204 if (!A)
2205 return false;
2206
2207 if (A->IsRegister) {
2208 Register Reg;
2209 if (parseNamedRegisterReference(PFS, Reg, Src: A->RegisterName.Value, Error)) {
2210 SourceRange = A->RegisterName.SourceRange;
2211 return true;
2212 }
2213 if (!RC.contains(Reg))
2214 return diagnoseRegisterClass(A->RegisterName);
2215 Arg = ArgDescriptor::createRegister(Reg);
2216 } else
2217 Arg = ArgDescriptor::createStack(Offset: A->StackOffset);
2218 // Check and apply the optional mask.
2219 if (A->Mask)
2220 Arg = ArgDescriptor::createArg(Arg, Mask: *A->Mask);
2221
2222 MFI->NumUserSGPRs += UserSGPRs;
2223 MFI->NumSystemSGPRs += SystemSGPRs;
2224 return false;
2225 };
2226
2227 if (YamlMFI.ArgInfo &&
2228 (parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentBuffer,
2229 AMDGPU::SGPR_128RegClass,
2230 MFI->ArgInfo.PrivateSegmentBuffer, 4, 0) ||
2231 parseAndCheckArgument(YamlMFI.ArgInfo->DispatchPtr,
2232 AMDGPU::SReg_64RegClass, MFI->ArgInfo.DispatchPtr,
2233 2, 0) ||
2234 parseAndCheckArgument(YamlMFI.ArgInfo->QueuePtr, AMDGPU::SReg_64RegClass,
2235 MFI->ArgInfo.QueuePtr, 2, 0) ||
2236 parseAndCheckArgument(YamlMFI.ArgInfo->KernargSegmentPtr,
2237 AMDGPU::SReg_64RegClass,
2238 MFI->ArgInfo.KernargSegmentPtr, 2, 0) ||
2239 parseAndCheckArgument(YamlMFI.ArgInfo->DispatchID,
2240 AMDGPU::SReg_64RegClass, MFI->ArgInfo.DispatchID,
2241 2, 0) ||
2242 parseAndCheckArgument(YamlMFI.ArgInfo->FlatScratchInit,
2243 AMDGPU::SReg_64RegClass,
2244 MFI->ArgInfo.FlatScratchInit, 2, 0) ||
2245 parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentSize,
2246 AMDGPU::SGPR_32RegClass,
2247 MFI->ArgInfo.PrivateSegmentSize, 1, 0) ||
2248 parseAndCheckArgument(YamlMFI.ArgInfo->LDSKernelId,
2249 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.LDSKernelId,
2250 1, 0) ||
2251 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDX,
2252 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDX,
2253 0, 1) ||
2254 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDY,
2255 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDY,
2256 0, 1) ||
2257 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDZ,
2258 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDZ,
2259 0, 1) ||
2260 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupInfo,
2261 AMDGPU::SGPR_32RegClass,
2262 MFI->ArgInfo.WorkGroupInfo, 0, 1) ||
2263 parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentWaveByteOffset,
2264 AMDGPU::SGPR_32RegClass,
2265 MFI->ArgInfo.PrivateSegmentWaveByteOffset, 0, 1) ||
2266 parseAndCheckArgument(YamlMFI.ArgInfo->ImplicitArgPtr,
2267 AMDGPU::SReg_64RegClass,
2268 MFI->ArgInfo.ImplicitArgPtr, 0, 0) ||
2269 parseAndCheckArgument(YamlMFI.ArgInfo->ImplicitBufferPtr,
2270 AMDGPU::SReg_64RegClass,
2271 MFI->ArgInfo.ImplicitBufferPtr, 2, 0) ||
2272 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDX,
2273 AMDGPU::VGPR_32RegClass, MFI->ArgInfo.WorkItemIDX,
2274 0, 0) ||
2275 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDY,
2276 AMDGPU::VGPR_32RegClass, MFI->ArgInfo.WorkItemIDY,
2277 0, 0) ||
2278 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDZ,
2279 AMDGPU::VGPR_32RegClass, MFI->ArgInfo.WorkItemIDZ,
2280 0, 0)))
2281 return true;
2282
2283 // Parse FirstKernArgPreloadReg separately, since it's a Register,
2284 // not ArgDescriptor.
2285 if (YamlMFI.ArgInfo && YamlMFI.ArgInfo->FirstKernArgPreloadReg) {
2286 const yaml::SIArgument &A = *YamlMFI.ArgInfo->FirstKernArgPreloadReg;
2287
2288 if (!A.IsRegister) {
2289 // For stack arguments, we don't have RegisterName.SourceRange,
2290 // but we should have some location info from the YAML parser
2291 const MemoryBuffer &Buffer =
2292 *PFS.SM->getMemoryBuffer(i: PFS.SM->getMainFileID());
2293 // Create a minimal valid source range
2294 SMLoc Loc = SMLoc::getFromPointer(Ptr: Buffer.getBufferStart());
2295 SMRange Range(Loc, Loc);
2296
2297 Error = SMDiagnostic(
2298 *PFS.SM, Loc, Buffer.getBufferIdentifier(), 1, 0, SourceMgr::DK_Error,
2299 "firstKernArgPreloadReg must be a register, not a stack location", "",
2300 {}, {});
2301
2302 SourceRange = Range;
2303 return true;
2304 }
2305
2306 Register Reg;
2307 if (parseNamedRegisterReference(PFS, Reg, Src: A.RegisterName.Value, Error)) {
2308 SourceRange = A.RegisterName.SourceRange;
2309 return true;
2310 }
2311
2312 if (!AMDGPU::SGPR_32RegClass.contains(Reg))
2313 return diagnoseRegisterClass(A.RegisterName);
2314
2315 MFI->ArgInfo.FirstKernArgPreloadReg = Reg;
2316 MFI->NumUserSGPRs += YamlMFI.NumKernargPreloadSGPRs;
2317 }
2318
2319 if (ST.hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode)) {
2320 MFI->Mode.IEEE = YamlMFI.Mode.IEEE;
2321 MFI->Mode.DX10Clamp = YamlMFI.Mode.DX10Clamp;
2322 }
2323
2324 // FIXME: Move proper support for denormal-fp-math into base MachineFunction
2325 MFI->Mode.FP32Denormals.Input = YamlMFI.Mode.FP32InputDenormals
2326 ? DenormalMode::IEEE
2327 : DenormalMode::PreserveSign;
2328 MFI->Mode.FP32Denormals.Output = YamlMFI.Mode.FP32OutputDenormals
2329 ? DenormalMode::IEEE
2330 : DenormalMode::PreserveSign;
2331
2332 MFI->Mode.FP64FP16Denormals.Input = YamlMFI.Mode.FP64FP16InputDenormals
2333 ? DenormalMode::IEEE
2334 : DenormalMode::PreserveSign;
2335 MFI->Mode.FP64FP16Denormals.Output = YamlMFI.Mode.FP64FP16OutputDenormals
2336 ? DenormalMode::IEEE
2337 : DenormalMode::PreserveSign;
2338
2339 if (YamlMFI.HasInitWholeWave)
2340 MFI->setInitWholeWave();
2341
2342 return false;
2343}
2344
2345//===----------------------------------------------------------------------===//
2346// AMDGPU CodeGen Pass Builder interface.
2347//===----------------------------------------------------------------------===//
2348
2349AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
2350 GCNTargetMachine &TM, const CGPassBuilderOption &Opts,
2351 PassInstrumentationCallbacks *PIC)
2352 : CodeGenPassBuilder(TM, Opts, PIC) {
2353 Opt.MISchedPostRA = true;
2354 Opt.RequiresCodeGenSCCOrder = true;
2355 // Exceptions and StackMaps are not supported, so these passes will never do
2356 // anything.
2357 // Garbage collection is not supported.
2358 disablePass<StackMapLivenessPass, FuncletLayoutPass, PatchableFunctionPass,
2359 ShadowStackGCLoweringPass, GCLoweringPass>();
2360}
2361
2362void AMDGPUCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
2363 if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN()) {
2364 flushFPMsToMPM(PMW);
2365 addModulePass(Pass: AMDGPURemoveIncompatibleFunctionsPass(TM), PMW);
2366 }
2367
2368 flushFPMsToMPM(PMW);
2369
2370 if (TM.getTargetTriple().isAMDGCN())
2371 addModulePass(Pass: AMDGPUPrintfRuntimeBindingPass(), PMW);
2372
2373 if (LowerCtorDtor)
2374 addModulePass(Pass: AMDGPUCtorDtorLoweringPass(), PMW);
2375
2376 if (isPassEnabled(Opt: EnableImageIntrinsicOptimizer))
2377 addFunctionPass(Pass: AMDGPUImageIntrinsicOptimizerPass(TM), PMW);
2378
2379 if (EnableUniformIntrinsicCombine)
2380 addFunctionPass(Pass: AMDGPUUniformIntrinsicCombinePass(), PMW);
2381 // This can be disabled by passing ::Disable here or on the command line
2382 // with --expand-variadics-override=disable.
2383 flushFPMsToMPM(PMW);
2384 addModulePass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Lowering), PMW);
2385
2386 addModulePass(Pass: AMDGPUAlwaysInlinePass(), PMW);
2387 addModulePass(Pass: AlwaysInlinerPass(), PMW);
2388
2389 addModulePass(Pass: AMDGPUExportKernelRuntimeHandlesPass(), PMW);
2390
2391 if (EnableLowerExecSync)
2392 addModulePass(Pass: AMDGPULowerExecSyncPass(), PMW);
2393
2394 if (EnableSwLowerLDS)
2395 addModulePass(Pass: AMDGPUSwLowerLDSPass(), PMW);
2396
2397 // Runs before PromoteAlloca so the latter can account for function uses
2398 if (EnableLowerModuleLDS)
2399 addModulePass(Pass: AMDGPULowerModuleLDSPass(getTM()), PMW);
2400
2401 // Run atomic optimizer before Atomic Expand
2402 if (TM.getOptLevel() >= CodeGenOptLevel::Less &&
2403 (AMDGPUAtomicOptimizerStrategy != ScanOptions::None))
2404 addFunctionPass(
2405 Pass: AMDGPUAtomicOptimizerPass(TM, AMDGPUAtomicOptimizerStrategy), PMW);
2406
2407 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
2408
2409 if (TM.getOptLevel() > CodeGenOptLevel::None) {
2410 addFunctionPass(Pass: AMDGPUPromoteAllocaPass(TM), PMW);
2411 if (isPassEnabled(Opt: EnableScalarIRPasses))
2412 addStraightLineScalarOptimizationPasses(PMW);
2413
2414 // TODO: Handle EnableAMDGPUAliasAnalysis
2415
2416 // TODO: May want to move later or split into an early and late one.
2417 addFunctionPass(Pass: AMDGPUCodeGenPreparePass(TM), PMW);
2418
2419 // Try to hoist loop invariant parts of divisions AMDGPUCodeGenPrepare may
2420 // have expanded.
2421 if (TM.getOptLevel() > CodeGenOptLevel::Less) {
2422 addFunctionPass(Pass: createFunctionToLoopPassAdaptor(Pass: LICMPass(LICMOptions()),
2423 /*UseMemorySSA=*/true),
2424 PMW);
2425 }
2426 }
2427
2428 Base::addIRPasses(PMW);
2429
2430 // EarlyCSE is not always strong enough to clean up what LSR produces. For
2431 // example, GVN can combine
2432 //
2433 // %0 = add %a, %b
2434 // %1 = add %b, %a
2435 //
2436 // and
2437 //
2438 // %0 = shl nsw %a, 2
2439 // %1 = shl %a, 2
2440 //
2441 // but EarlyCSE can do neither of them.
2442 if (isPassEnabled(Opt: EnableScalarIRPasses))
2443 addEarlyCSEOrGVNPass(PMW);
2444}
2445
2446void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(PassManagerWrapper &PMW) {
2447 if (TM.getOptLevel() > CodeGenOptLevel::None) {
2448 flushFPMsToMPM(PMW);
2449 addModulePass(Pass: AMDGPUPreloadKernelArgumentsPass(TM), PMW);
2450 }
2451
2452 if (EnableLowerKernelArguments)
2453 addFunctionPass(Pass: AMDGPULowerKernelArgumentsPass(TM), PMW);
2454
2455 Base::addCodeGenPrepare(PMW);
2456
2457 if (isPassEnabled(Opt: EnableLoadStoreVectorizer))
2458 addFunctionPass(Pass: LoadStoreVectorizerPass(), PMW);
2459
2460 // This lowering has been placed after codegenprepare to take advantage of
2461 // address mode matching (which is why it isn't put with the LDS lowerings).
2462 // It could be placed anywhere before uniformity annotations (an analysis
2463 // that it changes by splitting up fat pointers into their components)
2464 // but has been put before switch lowering and CFG flattening so that those
2465 // passes can run on the more optimized control flow this pass creates in
2466 // many cases.
2467 flushFPMsToMPM(PMW);
2468 addModulePass(Pass: AMDGPULowerBufferFatPointersPass(TM), PMW);
2469 flushFPMsToMPM(PMW);
2470 requireCGSCCOrder(PMW);
2471
2472 addModulePass(Pass: AMDGPULowerIntrinsicsPass(getTM()), PMW);
2473
2474 // LowerSwitch pass may introduce unreachable blocks that can cause unexpected
2475 // behavior for subsequent passes. Placing it here seems better that these
2476 // blocks would get cleaned up by UnreachableBlockElim inserted next in the
2477 // pass flow.
2478 addFunctionPass(Pass: LowerSwitchPass(), PMW);
2479}
2480
2481void AMDGPUCodeGenPassBuilder::addPreISel(PassManagerWrapper &PMW) {
2482
2483 if (TM.getOptLevel() > CodeGenOptLevel::None) {
2484 addFunctionPass(Pass: FlattenCFGPass(), PMW);
2485 addFunctionPass(Pass: SinkingPass(), PMW);
2486 addFunctionPass(Pass: AMDGPULateCodeGenPreparePass(getTM()), PMW);
2487 }
2488
2489 // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
2490 // regions formed by them.
2491
2492 addFunctionPass(Pass: AMDGPUUnifyDivergentExitNodesPass(), PMW);
2493 addFunctionPass(Pass: FixIrreduciblePass(), PMW);
2494 addFunctionPass(Pass: UnifyLoopExitsPass(), PMW);
2495 addFunctionPass(Pass: StructurizeCFGPass(/*SkipUniformRegions=*/false), PMW);
2496
2497 addFunctionPass(Pass: AMDGPUAnnotateUniformValuesPass(), PMW);
2498
2499 addFunctionPass(Pass: SIAnnotateControlFlowPass(getTM()), PMW);
2500
2501 // TODO: Move this right after structurizeCFG to avoid extra divergence
2502 // analysis. This depends on stopping SIAnnotateControlFlow from making
2503 // control flow modifications.
2504 addFunctionPass(Pass: AMDGPURewriteUndefForPHIPass(), PMW);
2505
2506 if (getCGPassBuilderOption().EnableGlobalISelOption !=
2507 cl::boolOrDefault::BOU_TRUE ||
2508 !isGlobalISelAbortEnabled())
2509 addFunctionPass(Pass: LCSSAPass(), PMW);
2510
2511 if (TM.getOptLevel() > CodeGenOptLevel::Less) {
2512 flushFPMsToMPM(PMW);
2513 addModulePass(Pass: AMDGPUPerfHintAnalysisPass(getTM()), PMW);
2514 }
2515}
2516
2517void AMDGPUCodeGenPassBuilder::addILPOpts(PassManagerWrapper &PMW) {
2518 if (EnableEarlyIfConversion)
2519 addMachineFunctionPass(Pass: EarlyIfConverterPass(), PMW);
2520
2521 Base::addILPOpts(PMW);
2522}
2523
2524void AMDGPUCodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
2525 addModulePass(Pass: AMDGPUAsmPrinterBeginPass(), PMW,
2526 /*Force=*/true);
2527}
2528
2529void AMDGPUCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
2530 addMachineFunctionPass(Pass: AMDGPUAsmPrinterPass(), PMW);
2531}
2532
2533void AMDGPUCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
2534 addModulePass(Pass: AMDGPUAsmPrinterEndPass(), PMW);
2535}
2536
2537Error AMDGPUCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
2538 addMachineFunctionPass(Pass: AMDGPUISelDAGToDAGPass(TM), PMW);
2539 addMachineFunctionPass(Pass: SIFixSGPRCopiesPass(), PMW);
2540 addMachineFunctionPass(Pass: SILowerI1CopiesPass(), PMW);
2541 return Error::success();
2542}
2543
2544Error AMDGPUCodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
2545 addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW);
2546 return Error::success();
2547}
2548
2549void AMDGPUCodeGenPassBuilder::addPreLegalizeMachineIR(
2550 PassManagerWrapper &PMW) {
2551 addMachineFunctionPass(Pass: AMDGPUPreLegalizerCombinerPass(), PMW);
2552 addMachineFunctionPass(Pass: LocalizerPass(), PMW);
2553}
2554
2555Error AMDGPUCodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
2556 addMachineFunctionPass(Pass: LegalizerPass(), PMW);
2557 return Error::success();
2558}
2559
2560void AMDGPUCodeGenPassBuilder::addPreRegBankSelect(PassManagerWrapper &PMW) {
2561 addMachineFunctionPass(Pass: AMDGPUPostLegalizerCombinerPass(), PMW);
2562 addMachineFunctionPass(Pass: AMDGPUGlobalISelDivergenceLoweringPass(), PMW);
2563}
2564
2565Error AMDGPUCodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
2566 addMachineFunctionPass(Pass: AMDGPURegBankSelectPass(), PMW);
2567 addMachineFunctionPass(Pass: AMDGPURegBankLegalizePass(), PMW);
2568 return Error::success();
2569}
2570
2571void AMDGPUCodeGenPassBuilder::addPreGlobalInstructionSelect(
2572 PassManagerWrapper &PMW) {
2573 bool IsOptLevelNone = getOptLevel() == CodeGenOptLevel::None;
2574 addMachineFunctionPass(Pass: AMDGPURegBankCombinerPass(IsOptLevelNone), PMW);
2575}
2576
2577Error AMDGPUCodeGenPassBuilder::addGlobalInstructionSelect(
2578 PassManagerWrapper &PMW) {
2579 addMachineFunctionPass(Pass: InstructionSelectPass(getOptLevel()), PMW);
2580 return Error::success();
2581}
2582
2583void AMDGPUCodeGenPassBuilder::addPreRewrite(PassManagerWrapper &PMW) {
2584 if (EnableRegReassign) {
2585 addMachineFunctionPass(Pass: GCNNSAReassignPass(), PMW);
2586 }
2587
2588 addMachineFunctionPass(Pass: AMDGPURewriteAGPRCopyMFMAPass(), PMW);
2589}
2590
2591void AMDGPUCodeGenPassBuilder::addMachineSSAOptimization(
2592 PassManagerWrapper &PMW) {
2593 Base::addMachineSSAOptimization(PMW);
2594
2595 addMachineFunctionPass(Pass: SIFoldOperandsPass(), PMW);
2596 if (EnableDPPCombine) {
2597 addMachineFunctionPass(Pass: GCNDPPCombinePass(), PMW);
2598 }
2599 addMachineFunctionPass(Pass: SILoadStoreOptimizerPass(), PMW);
2600 if (isPassEnabled(Opt: EnableSDWAPeephole)) {
2601 addMachineFunctionPass(Pass: SIPeepholeSDWAPass(), PMW);
2602 addMachineFunctionPass(Pass: EarlyMachineLICMPass(), PMW);
2603 addMachineFunctionPass(Pass: MachineCSEPass(), PMW);
2604 addMachineFunctionPass(Pass: SIFoldOperandsPass(), PMW);
2605 }
2606 addMachineFunctionPass(Pass: DeadMachineInstructionElimPass(), PMW);
2607 addMachineFunctionPass(Pass: SIShrinkInstructionsPass(), PMW);
2608}
2609
2610Error AMDGPUCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
2611 insertPass<PHIEliminationPass>(Pass: SILowerControlFlowPass());
2612
2613 insertPass<TwoAddressInstructionPass>(Pass: SIWholeQuadModePass());
2614
2615 return Base::addFastRegAlloc(PMW);
2616}
2617
2618Error AMDGPUCodeGenPassBuilder::addRegAssignAndRewriteFast(
2619 PassManagerWrapper &PMW) {
2620 if (auto Err = validateRegAllocOptions())
2621 return Err;
2622
2623 addMachineFunctionPass(Pass: GCNPreRALongBranchRegPass(), PMW);
2624
2625 // SGPR allocation - default to fast at -O0.
2626 if (SGPRRegAllocNPM == RegAllocType::Greedy)
2627 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateSGPRs, "sgpr"}), PMW);
2628 else
2629 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateSGPRs, "sgpr", false}),
2630 PMW);
2631
2632 // Equivalent of PEI for SGPRs.
2633 addMachineFunctionPass(Pass: SILowerSGPRSpillsPass(), PMW);
2634
2635 // To Allocate wwm registers used in whole quad mode operations (for shaders).
2636 addMachineFunctionPass(Pass: SIPreAllocateWWMRegsPass(), PMW);
2637
2638 // WWM allocation - default to fast at -O0.
2639 if (WWMRegAllocNPM == RegAllocType::Greedy)
2640 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateWWMRegs, "wwm"}), PMW);
2641 else
2642 addMachineFunctionPass(
2643 Pass: RegAllocFastPass({onlyAllocateWWMRegs, "wwm", false}), PMW);
2644
2645 addMachineFunctionPass(Pass: SILowerWWMCopiesPass(), PMW);
2646 addMachineFunctionPass(Pass: AMDGPUReserveWWMRegsPass(), PMW);
2647
2648 // VGPR allocation - default to fast at -O0.
2649 if (VGPRRegAllocNPM == RegAllocType::Greedy)
2650 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2651 else
2652 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2653
2654 return Error::success();
2655}
2656
2657Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
2658 if (EnableDCEInRA)
2659 insertPass<DetectDeadLanesPass>(Pass: DeadMachineInstructionElimPass());
2660
2661 // FIXME: when an instruction has a Killed operand, and the instruction is
2662 // inside a bundle, seems only the BUNDLE instruction appears as the Kills of
2663 // the register in LiveVariables, this would trigger a failure in verifier,
2664 // we should fix it and enable the verifier.
2665 if (OptVGPRLiveRange)
2666 insertPass<RequireAnalysisPass<LiveVariablesAnalysis, MachineFunction>>(
2667 Pass: SIOptimizeVGPRLiveRangePass());
2668
2669 // This must be run immediately after phi elimination and before
2670 // TwoAddressInstructions, otherwise the processing of the tied operand of
2671 // SI_ELSE will introduce a copy of the tied operand source after the else.
2672 insertPass<PHIEliminationPass>(Pass: SILowerControlFlowPass());
2673
2674 if (EnableRewritePartialRegUses)
2675 insertPass<RenameIndependentSubregsPass>(Pass: GCNRewritePartialRegUsesPass());
2676
2677 if (isPassEnabled(Opt: EnablePreRAOptimizations))
2678 insertPass<MachineSchedulerPass>(Pass: GCNPreRAOptimizationsPass());
2679
2680 // Allow the scheduler to run before SIWholeQuadMode inserts exec manipulation
2681 // instructions that cause scheduling barriers.
2682 insertPass<MachineSchedulerPass>(Pass: SIWholeQuadModePass());
2683
2684 if (OptExecMaskPreRA)
2685 insertPass<MachineSchedulerPass>(Pass: SIOptimizeExecMaskingPreRAPass());
2686
2687 // This is not an essential optimization and it has a noticeable impact on
2688 // compilation time, so we only enable it from O2.
2689 if (TM.getOptLevel() > CodeGenOptLevel::Less)
2690 insertPass<MachineSchedulerPass>(Pass: SIFormMemoryClausesPass());
2691
2692 return Base::addOptimizedRegAlloc(PMW);
2693}
2694
2695void AMDGPUCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
2696 if (getOptLevel() != CodeGenOptLevel::None)
2697 addMachineFunctionPass(Pass: AMDGPUPrepareAGPRAllocPass(), PMW);
2698 if (getOptLevel() >= CodeGenOptLevel::Default && EnableMachinePipeliner)
2699 addMachineFunctionPass(Pass: MachinePipelinerPass(), PMW);
2700}
2701
2702Expected<bool> AMDGPUCodeGenPassBuilder::addRegAssignAndRewriteOptimized(
2703 PassManagerWrapper &PMW) {
2704 if (auto Err = validateRegAllocOptions())
2705 return Err;
2706
2707 addMachineFunctionPass(Pass: GCNPreRALongBranchRegPass(), PMW);
2708
2709 // SGPR allocation - default to greedy at -O1 and above.
2710 if (SGPRRegAllocNPM == RegAllocType::Fast)
2711 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateSGPRs, "sgpr", false}),
2712 PMW);
2713 else
2714 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateSGPRs, "sgpr"}), PMW);
2715
2716 // Commit allocated register changes. This is mostly necessary because too
2717 // many things rely on the use lists of the physical registers, such as the
2718 // verifier. This is only necessary with allocators which use LiveIntervals,
2719 // since FastRegAlloc does the replacements itself.
2720 addMachineFunctionPass(Pass: VirtRegRewriterPass(false), PMW);
2721
2722 // At this point, the sgpr-regalloc has been done and it is good to have the
2723 // stack slot coloring to try to optimize the SGPR spill stack indices before
2724 // attempting the custom SGPR spill lowering.
2725 addMachineFunctionPass(Pass: StackSlotColoringPass(), PMW);
2726
2727 // Equivalent of PEI for SGPRs.
2728 addMachineFunctionPass(Pass: SILowerSGPRSpillsPass(), PMW);
2729
2730 // To Allocate wwm registers used in whole quad mode operations (for shaders).
2731 addMachineFunctionPass(Pass: SIPreAllocateWWMRegsPass(), PMW);
2732
2733 // WWM allocation - default to greedy at -O1 and above.
2734 if (WWMRegAllocNPM == RegAllocType::Fast)
2735 addMachineFunctionPass(
2736 Pass: RegAllocFastPass({onlyAllocateWWMRegs, "wwm", false}), PMW);
2737 else
2738 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateWWMRegs, "wwm"}), PMW);
2739 addMachineFunctionPass(Pass: SILowerWWMCopiesPass(), PMW);
2740 addMachineFunctionPass(Pass: VirtRegRewriterPass(false), PMW);
2741 addMachineFunctionPass(Pass: AMDGPUReserveWWMRegsPass(), PMW);
2742
2743 // VGPR allocation - default to greedy at -O1 and above.
2744 if (VGPRRegAllocNPM == RegAllocType::Fast)
2745 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2746 else
2747 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2748
2749 addPreRewrite(PMW);
2750 addMachineFunctionPass(Pass: VirtRegRewriterPass(true), PMW);
2751
2752 addMachineFunctionPass(Pass: AMDGPUMarkLastScratchLoadPass(), PMW);
2753 return true;
2754}
2755
2756void AMDGPUCodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) {
2757 addMachineFunctionPass(Pass: SIFixVGPRCopiesPass(), PMW);
2758 if (TM.getOptLevel() > CodeGenOptLevel::None)
2759 addMachineFunctionPass(Pass: SIOptimizeExecMaskingPass(), PMW);
2760 Base::addPostRegAlloc(PMW);
2761}
2762
2763void AMDGPUCodeGenPassBuilder::addPreSched2(PassManagerWrapper &PMW) {
2764 if (TM.getOptLevel() > CodeGenOptLevel::None)
2765 addMachineFunctionPass(Pass: SIShrinkInstructionsPass(), PMW);
2766 addMachineFunctionPass(Pass: SIPostRABundlerPass(), PMW);
2767}
2768
2769void AMDGPUCodeGenPassBuilder::addPostBBSections(PassManagerWrapper &PMW) {
2770 // We run this later to avoid passes like livedebugvalues and BBSections
2771 // having to deal with the apparent multi-entry functions we may generate.
2772 addMachineFunctionPass(Pass: AMDGPUPreloadKernArgPrologPass(), PMW);
2773}
2774
2775void AMDGPUCodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
2776 if (isPassEnabled(Opt: EnableVOPD, Level: CodeGenOptLevel::Less)) {
2777 addMachineFunctionPass(Pass: GCNCreateVOPDPass(), PMW);
2778 }
2779
2780 addMachineFunctionPass(Pass: SIMemoryLegalizerPass(), PMW);
2781 addMachineFunctionPass(Pass: SIInsertWaitcntsPass(), PMW);
2782
2783 addMachineFunctionPass(Pass: SIModeRegisterPass(), PMW);
2784
2785 if (TM.getOptLevel() > CodeGenOptLevel::None)
2786 addMachineFunctionPass(Pass: SIInsertHardClausesPass(), PMW);
2787
2788 addMachineFunctionPass(Pass: SILateBranchLoweringPass(), PMW);
2789
2790 if (isPassEnabled(Opt: EnableSetWavePriority, Level: CodeGenOptLevel::Less))
2791 addMachineFunctionPass(Pass: AMDGPUSetWavePriorityPass(), PMW);
2792
2793 if (TM.getOptLevel() > CodeGenOptLevel::None)
2794 addMachineFunctionPass(Pass: SIPreEmitPeepholePass(), PMW);
2795
2796 // The hazard recognizer that runs as part of the post-ra scheduler does not
2797 // guarantee to be able handle all hazards correctly. This is because if there
2798 // are multiple scheduling regions in a basic block, the regions are scheduled
2799 // bottom up, so when we begin to schedule a region we don't know what
2800 // instructions were emitted directly before it.
2801 //
2802 // Here we add a stand-alone hazard recognizer pass which can handle all
2803 // cases.
2804 addMachineFunctionPass(Pass: PostRAHazardRecognizerPass(), PMW);
2805 addMachineFunctionPass(Pass: AMDGPUWaitSGPRHazardsPass(), PMW);
2806 addMachineFunctionPass(Pass: AMDGPULowerVGPREncodingPass(), PMW);
2807
2808 if (isPassEnabled(Opt: EnableInsertDelayAlu, Level: CodeGenOptLevel::Less)) {
2809 addMachineFunctionPass(Pass: AMDGPUInsertDelayAluPass(), PMW);
2810 }
2811
2812 addMachineFunctionPass(Pass: BranchRelaxationPass(), PMW);
2813}
2814
2815bool AMDGPUCodeGenPassBuilder::isPassEnabled(const cl::opt<bool> &Opt,
2816 CodeGenOptLevel Level) const {
2817 if (Opt.getNumOccurrences())
2818 return Opt;
2819 if (TM.getOptLevel() < Level)
2820 return false;
2821 return Opt;
2822}
2823
2824void AMDGPUCodeGenPassBuilder::addEarlyCSEOrGVNPass(PassManagerWrapper &PMW) {
2825 if (TM.getOptLevel() == CodeGenOptLevel::Aggressive)
2826 addFunctionPass(Pass: GVNPass(), PMW);
2827 else
2828 addFunctionPass(Pass: EarlyCSEPass(), PMW);
2829}
2830
2831void AMDGPUCodeGenPassBuilder::addStraightLineScalarOptimizationPasses(
2832 PassManagerWrapper &PMW) {
2833 if (isPassEnabled(Opt: EnableLoopPrefetch, Level: CodeGenOptLevel::Aggressive))
2834 addFunctionPass(Pass: LoopDataPrefetchPass(), PMW);
2835
2836 addFunctionPass(Pass: SeparateConstOffsetFromGEPPass(), PMW);
2837
2838 // ReassociateGEPs exposes more opportunities for SLSR. See
2839 // the example in reassociate-geps-and-slsr.ll.
2840 addFunctionPass(Pass: StraightLineStrengthReducePass(), PMW);
2841
2842 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or
2843 // EarlyCSE can reuse.
2844 addEarlyCSEOrGVNPass(PMW);
2845
2846 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
2847 addFunctionPass(Pass: NaryReassociatePass(), PMW);
2848
2849 // NaryReassociate on GEPs creates redundant common expressions, so run
2850 // EarlyCSE after it.
2851 addFunctionPass(Pass: EarlyCSEPass(), PMW);
2852}
2853