1//===-- AMDGPUTargetMachine.cpp - TargetMachine for hw codegen targets-----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file contains both AMDGPU target machine and the CodeGen pass builder.
11/// The AMDGPU target machine contains all of the hardware specific information
12/// needed to emit code for SI+ GPUs in the legacy pass manager pipeline. The
13/// CodeGen pass builder handles the pass pipeline for new pass manager.
14//
15//===----------------------------------------------------------------------===//
16
17#include "AMDGPUTargetMachine.h"
18#include "AMDGPU.h"
19#include "AMDGPUAliasAnalysis.h"
20#include "AMDGPUAsmPrinter.h"
21#include "AMDGPUBarrierLatency.h"
22#include "AMDGPUCoExecSchedStrategy.h"
23#include "AMDGPUCtorDtorLowering.h"
24#include "AMDGPUExportClustering.h"
25#include "AMDGPUExportKernelRuntimeHandles.h"
26#include "AMDGPUHazardLatency.h"
27#include "AMDGPUIGroupLP.h"
28#include "AMDGPUISelDAGToDAG.h"
29#include "AMDGPULowerVGPREncoding.h"
30#include "AMDGPUMacroFusion.h"
31#include "AMDGPUNextUseAnalysis.h"
32#include "AMDGPUPerfHintAnalysis.h"
33#include "AMDGPUPreloadKernArgProlog.h"
34#include "AMDGPUPrepareAGPRAlloc.h"
35#include "AMDGPURemoveIncompatibleFunctions.h"
36#include "AMDGPUReserveWWMRegs.h"
37#include "AMDGPUResourceUsageAnalysis.h"
38#include "AMDGPUSplitModule.h"
39#include "AMDGPUTargetObjectFile.h"
40#include "AMDGPUTargetTransformInfo.h"
41#include "AMDGPUUnifyDivergentExitNodes.h"
42#include "AMDGPUWaitSGPRHazards.h"
43#include "GCNDPPCombine.h"
44#include "GCNIterativeScheduler.h"
45#include "GCNNSAReassign.h"
46#include "GCNPreRALongBranchReg.h"
47#include "GCNPreRAOptimizations.h"
48#include "GCNRewritePartialRegUses.h"
49#include "GCNSchedStrategy.h"
50#include "GCNVOPDUtils.h"
51#include "R600.h"
52#include "R600TargetMachine.h"
53#include "SIFixSGPRCopies.h"
54#include "SIFixVGPRCopies.h"
55#include "SIFoldOperands.h"
56#include "SIFormMemoryClauses.h"
57#include "SILoadStoreOptimizer.h"
58#include "SILowerControlFlow.h"
59#include "SILowerSGPRSpills.h"
60#include "SILowerWWMCopies.h"
61#include "SIMachineFunctionInfo.h"
62#include "SIMachineScheduler.h"
63#include "SIOptimizeExecMasking.h"
64#include "SIOptimizeExecMaskingPreRA.h"
65#include "SIOptimizeVGPRLiveRange.h"
66#include "SIPeepholeSDWA.h"
67#include "SIPostRABundler.h"
68#include "SIPreAllocateWWMRegs.h"
69#include "SIShrinkInstructions.h"
70#include "SIWholeQuadMode.h"
71#include "TargetInfo/AMDGPUTargetInfo.h"
72#include "Utils/AMDGPUBaseInfo.h"
73#include "llvm/Analysis/CGSCCPassManager.h"
74#include "llvm/Analysis/KernelInfo.h"
75#include "llvm/CodeGen/AtomicExpand.h"
76#include "llvm/CodeGen/BranchRelaxation.h"
77#include "llvm/CodeGen/DeadMachineInstructionElim.h"
78#include "llvm/CodeGen/DetectDeadLanes.h"
79#include "llvm/CodeGen/EarlyIfConversion.h"
80#include "llvm/CodeGen/FuncletLayout.h"
81#include "llvm/CodeGen/GCMetadata.h"
82#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
83#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
84#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
85#include "llvm/CodeGen/GlobalISel/Legalizer.h"
86#include "llvm/CodeGen/GlobalISel/Localizer.h"
87#include "llvm/CodeGen/MIRParser/MIParser.h"
88#include "llvm/CodeGen/MachineCSE.h"
89#include "llvm/CodeGen/MachineLICM.h"
90#include "llvm/CodeGen/MachinePipeliner.h"
91#include "llvm/CodeGen/MachineScheduler.h"
92#include "llvm/CodeGen/PHIElimination.h"
93#include "llvm/CodeGen/Passes.h"
94#include "llvm/CodeGen/PatchableFunction.h"
95#include "llvm/CodeGen/PostRAHazardRecognizer.h"
96#include "llvm/CodeGen/RegAllocGreedyPass.h"
97#include "llvm/CodeGen/RegAllocRegistry.h"
98#include "llvm/CodeGen/RenameIndependentSubregs.h"
99#include "llvm/CodeGen/ShadowStackGCLowering.h"
100#include "llvm/CodeGen/StackSlotColoring.h"
101#include "llvm/CodeGen/TargetPassConfig.h"
102#include "llvm/CodeGen/TwoAddressInstructionPass.h"
103#include "llvm/IR/DiagnosticInfo.h"
104#include "llvm/IR/IntrinsicsAMDGPU.h"
105#include "llvm/IR/Module.h"
106#include "llvm/IR/PassManager.h"
107#include "llvm/IR/PatternMatch.h"
108#include "llvm/InitializePasses.h"
109#include "llvm/MC/TargetRegistry.h"
110#include "llvm/Option/LibraryOptions.h"
111#include "llvm/Passes/CodeGenPassBuilder.h"
112#include "llvm/Passes/PassBuilder.h"
113#include "llvm/Support/Compiler.h"
114#include "llvm/Support/FormatVariadic.h"
115#include "llvm/TargetParser/AMDGPUTargetParser.h"
116#include "llvm/Transforms/HipStdPar/HipStdPar.h"
117#include "llvm/Transforms/IPO.h"
118#include "llvm/Transforms/IPO/AlwaysInliner.h"
119#include "llvm/Transforms/IPO/ExpandVariadics.h"
120#include "llvm/Transforms/IPO/GlobalDCE.h"
121#include "llvm/Transforms/IPO/Internalize.h"
122#include "llvm/Transforms/Scalar.h"
123#include "llvm/Transforms/Scalar/EarlyCSE.h"
124#include "llvm/Transforms/Scalar/FlattenCFG.h"
125#include "llvm/Transforms/Scalar/GVN.h"
126#include "llvm/Transforms/Scalar/InferAddressSpaces.h"
127#include "llvm/Transforms/Scalar/LICM.h"
128#include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
129#include "llvm/Transforms/Scalar/LoopPassManager.h"
130#include "llvm/Transforms/Scalar/NaryReassociate.h"
131#include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"
132#include "llvm/Transforms/Scalar/Sink.h"
133#include "llvm/Transforms/Scalar/StraightLineStrengthReduce.h"
134#include "llvm/Transforms/Scalar/StructurizeCFG.h"
135#include "llvm/Transforms/Utils.h"
136#include "llvm/Transforms/Utils/FixIrreducible.h"
137#include "llvm/Transforms/Utils/LCSSA.h"
138#include "llvm/Transforms/Utils/LowerSwitch.h"
139#include "llvm/Transforms/Utils/SimplifyLibCalls.h"
140#include "llvm/Transforms/Utils/UnifyLoopExits.h"
141#include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
142#include <optional>
143
144using namespace llvm;
145using namespace llvm::PatternMatch;
146
147namespace {
148//===----------------------------------------------------------------------===//
149// AMDGPU CodeGen Pass Builder interface.
150//===----------------------------------------------------------------------===//
151
152class AMDGPUCodeGenPassBuilder : public CodeGenPassBuilder {
153 using Base = CodeGenPassBuilder;
154
155 const AMDGPUOptions &CLOpts;
156
157 GCNTargetMachine &getTM() const {
158 return static_cast<GCNTargetMachine &>(TM);
159 }
160
161public:
162 AMDGPUCodeGenPassBuilder(GCNTargetMachine &TM,
163 const CGPassBuilderOption &Opts,
164 PassInstrumentationCallbacks *PIC);
165
166 void addIRPasses(PassManagerWrapper &PMW) override;
167 void addCodeGenPrepare(PassManagerWrapper &PMW) override;
168 void addPreISel(PassManagerWrapper &PMW) override;
169 void addILPOpts(PassManagerWrapper &PMW) override;
170 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
171 void addAsmPrinter(PassManagerWrapper &PMW) override;
172 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
173 Error addInstSelector(PassManagerWrapper &PMW) override;
174 Error addIRTranslator(PassManagerWrapper &PMW) override;
175 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
176 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
177 void addPreRegBankSelect(PassManagerWrapper &PMW) override;
178 Error addRegBankSelect(PassManagerWrapper &PMW) override;
179 void addPreGlobalInstructionSelect(PassManagerWrapper &PMW) override;
180 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
181 void addPreRewrite(PassManagerWrapper &PMW) override;
182 void addMachineSSAOptimization(PassManagerWrapper &PMW) override;
183 void addPostRegAlloc(PassManagerWrapper &PMW) override;
184 void addPreEmitPass(PassManagerWrapper &PMW) override;
185 Error addRegAssignAndRewriteFast(PassManagerWrapper &PMW) override;
186 Expected<bool>
187 addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) override;
188 void addPreRegAlloc(PassManagerWrapper &PMW) override;
189 Error addFastRegAlloc(PassManagerWrapper &PMW) override;
190 Error addOptimizedRegAlloc(PassManagerWrapper &PMW) override;
191 void addPreSched2(PassManagerWrapper &PMW) override;
192 void addPostBBSections(PassManagerWrapper &PMW) override;
193
194private:
195 Error validateRegAllocOptions() const;
196
197public:
198 /// Check if a pass is enabled given \p Opt option. The option always
199 /// overrides defaults if explicitly used. Otherwise the pass is enabled at an
200 /// optimization \p Level minimum.
201 bool isPassEnabled(BoolOrDefault Opt,
202 CodeGenOptLevel Level = CodeGenOptLevel::Default) const;
203 void addEarlyCSEOrGVNPass(PassManagerWrapper &PMW);
204 void addStraightLineScalarOptimizationPasses(PassManagerWrapper &PMW);
205};
206
207class SGPRRegisterRegAlloc : public RegisterRegAllocBase<SGPRRegisterRegAlloc> {
208public:
209 SGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
210 : RegisterRegAllocBase(N, D, C) {}
211};
212
213class VGPRRegisterRegAlloc : public RegisterRegAllocBase<VGPRRegisterRegAlloc> {
214public:
215 VGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
216 : RegisterRegAllocBase(N, D, C) {}
217};
218
219class WWMRegisterRegAlloc : public RegisterRegAllocBase<WWMRegisterRegAlloc> {
220public:
221 WWMRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
222 : RegisterRegAllocBase(N, D, C) {}
223};
224
225static bool onlyAllocateSGPRs(const TargetRegisterInfo &TRI,
226 const MachineRegisterInfo &MRI,
227 const Register Reg) {
228 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
229 return static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC);
230}
231
232static bool onlyAllocateVGPRs(const TargetRegisterInfo &TRI,
233 const MachineRegisterInfo &MRI,
234 const Register Reg) {
235 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
236 return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC);
237}
238
239static bool onlyAllocateWWMRegs(const TargetRegisterInfo &TRI,
240 const MachineRegisterInfo &MRI,
241 const Register Reg) {
242 const SIMachineFunctionInfo *MFI =
243 MRI.getMF().getInfo<SIMachineFunctionInfo>();
244 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
245 return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC) &&
246 MFI->checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG);
247}
248
249/// -{sgpr|wwm|vgpr}-regalloc=... command line option.
250static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
251
252/// A dummy default pass factory indicates whether the register allocator is
253/// overridden on the command line.
254static llvm::once_flag InitializeDefaultSGPRRegisterAllocatorFlag;
255static llvm::once_flag InitializeDefaultVGPRRegisterAllocatorFlag;
256static llvm::once_flag InitializeDefaultWWMRegisterAllocatorFlag;
257
258static SGPRRegisterRegAlloc
259defaultSGPRRegAlloc("default",
260 "pick SGPR register allocator based on -O option",
261 useDefaultRegisterAllocator);
262
263static cl::opt<SGPRRegisterRegAlloc::FunctionPassCtor, false,
264 RegisterPassParser<SGPRRegisterRegAlloc>>
265SGPRRegAlloc("sgpr-regalloc", cl::Hidden, cl::init(Val: &useDefaultRegisterAllocator),
266 cl::desc("Register allocator to use for SGPRs"));
267
268static cl::opt<VGPRRegisterRegAlloc::FunctionPassCtor, false,
269 RegisterPassParser<VGPRRegisterRegAlloc>>
270VGPRRegAlloc("vgpr-regalloc", cl::Hidden, cl::init(Val: &useDefaultRegisterAllocator),
271 cl::desc("Register allocator to use for VGPRs"));
272
273static cl::opt<WWMRegisterRegAlloc::FunctionPassCtor, false,
274 RegisterPassParser<WWMRegisterRegAlloc>>
275 WWMRegAlloc("wwm-regalloc", cl::Hidden,
276 cl::init(Val: &useDefaultRegisterAllocator),
277 cl::desc("Register allocator to use for WWM registers"));
278
279// New pass manager register allocator options for AMDGPU
280static cl::opt<RegAllocType, false, RegAllocTypeParser> SGPRRegAllocNPM(
281 "sgpr-regalloc-npm", cl::Hidden, cl::init(Val: RegAllocType::Default),
282 cl::desc("Register allocator for SGPRs (new pass manager)"));
283
284static cl::opt<RegAllocType, false, RegAllocTypeParser> VGPRRegAllocNPM(
285 "vgpr-regalloc-npm", cl::Hidden, cl::init(Val: RegAllocType::Default),
286 cl::desc("Register allocator for VGPRs (new pass manager)"));
287
288static cl::opt<RegAllocType, false, RegAllocTypeParser> WWMRegAllocNPM(
289 "wwm-regalloc-npm", cl::Hidden, cl::init(Val: RegAllocType::Default),
290 cl::desc("Register allocator for WWM registers (new pass manager)"));
291
292/// Check if the given RegAllocType is supported for AMDGPU NPM register
293/// allocation. Only Fast and Greedy are supported; Basic and PBQP are not.
294static Error checkRegAllocSupported(RegAllocType RAType, StringRef RegName) {
295 if (RAType == RegAllocType::Basic || RAType == RegAllocType::PBQP) {
296 return make_error<StringError>(
297 Args: Twine("unsupported register allocator '") +
298 (RAType == RegAllocType::Basic ? "basic" : "pbqp") + "' for " +
299 RegName + " registers",
300 Args: inconvertibleErrorCode());
301 }
302 return Error::success();
303}
304
305Error AMDGPUCodeGenPassBuilder::validateRegAllocOptions() const {
306 // 1. Generic --regalloc-npm is not supported for AMDGPU.
307 if (Opt.RegAlloc != RegAllocType::Unset) {
308 return make_error<StringError>(
309 Args: "-regalloc-npm not supported for amdgcn. Use -sgpr-regalloc-npm, "
310 "-vgpr-regalloc-npm, and -wwm-regalloc-npm",
311 Args: inconvertibleErrorCode());
312 }
313
314 // 2. Legacy PM regalloc options are not compatible with NPM.
315 if (SGPRRegAlloc.getNumOccurrences() > 0 ||
316 VGPRRegAlloc.getNumOccurrences() > 0 ||
317 WWMRegAlloc.getNumOccurrences() > 0) {
318 return make_error<StringError>(
319 Args: "-sgpr-regalloc, -vgpr-regalloc, and -wwm-regalloc are legacy PM "
320 "options. Use -sgpr-regalloc-npm, -vgpr-regalloc-npm, and "
321 "-wwm-regalloc-npm with the new pass manager",
322 Args: inconvertibleErrorCode());
323 }
324
325 // 3. Only Fast and Greedy allocators are supported for AMDGPU.
326 if (auto Err = checkRegAllocSupported(RAType: SGPRRegAllocNPM, RegName: "SGPR"))
327 return Err;
328 if (auto Err = checkRegAllocSupported(RAType: WWMRegAllocNPM, RegName: "WWM"))
329 return Err;
330 if (auto Err = checkRegAllocSupported(RAType: VGPRRegAllocNPM, RegName: "VGPR"))
331 return Err;
332
333 return Error::success();
334}
335
336static void initializeDefaultSGPRRegisterAllocatorOnce() {
337 RegisterRegAlloc::FunctionPassCtor Ctor = SGPRRegisterRegAlloc::getDefault();
338
339 if (!Ctor) {
340 Ctor = SGPRRegAlloc;
341 SGPRRegisterRegAlloc::setDefault(SGPRRegAlloc);
342 }
343}
344
345static void initializeDefaultVGPRRegisterAllocatorOnce() {
346 RegisterRegAlloc::FunctionPassCtor Ctor = VGPRRegisterRegAlloc::getDefault();
347
348 if (!Ctor) {
349 Ctor = VGPRRegAlloc;
350 VGPRRegisterRegAlloc::setDefault(VGPRRegAlloc);
351 }
352}
353
354static void initializeDefaultWWMRegisterAllocatorOnce() {
355 RegisterRegAlloc::FunctionPassCtor Ctor = WWMRegisterRegAlloc::getDefault();
356
357 if (!Ctor) {
358 Ctor = WWMRegAlloc;
359 WWMRegisterRegAlloc::setDefault(WWMRegAlloc);
360 }
361}
362
363static FunctionPass *createBasicSGPRRegisterAllocator() {
364 return createBasicRegisterAllocator(F: onlyAllocateSGPRs);
365}
366
367static FunctionPass *createGreedySGPRRegisterAllocator() {
368 return createGreedyRegisterAllocator(F: onlyAllocateSGPRs);
369}
370
371static FunctionPass *createFastSGPRRegisterAllocator() {
372 return createFastRegisterAllocator(F: onlyAllocateSGPRs, ClearVirtRegs: false);
373}
374
375static FunctionPass *createBasicVGPRRegisterAllocator() {
376 return createBasicRegisterAllocator(F: onlyAllocateVGPRs);
377}
378
379static FunctionPass *createGreedyVGPRRegisterAllocator() {
380 return createGreedyRegisterAllocator(F: onlyAllocateVGPRs);
381}
382
383static FunctionPass *createFastVGPRRegisterAllocator() {
384 return createFastRegisterAllocator(F: onlyAllocateVGPRs, ClearVirtRegs: true);
385}
386
387static FunctionPass *createBasicWWMRegisterAllocator() {
388 return createBasicRegisterAllocator(F: onlyAllocateWWMRegs);
389}
390
391static FunctionPass *createGreedyWWMRegisterAllocator() {
392 return createGreedyRegisterAllocator(F: onlyAllocateWWMRegs);
393}
394
395static FunctionPass *createFastWWMRegisterAllocator() {
396 return createFastRegisterAllocator(F: onlyAllocateWWMRegs, ClearVirtRegs: false);
397}
398
399static SGPRRegisterRegAlloc basicRegAllocSGPR(
400 "basic", "basic register allocator", createBasicSGPRRegisterAllocator);
401static SGPRRegisterRegAlloc greedyRegAllocSGPR(
402 "greedy", "greedy register allocator", createGreedySGPRRegisterAllocator);
403
404static SGPRRegisterRegAlloc fastRegAllocSGPR(
405 "fast", "fast register allocator", createFastSGPRRegisterAllocator);
406
407
408static VGPRRegisterRegAlloc basicRegAllocVGPR(
409 "basic", "basic register allocator", createBasicVGPRRegisterAllocator);
410static VGPRRegisterRegAlloc greedyRegAllocVGPR(
411 "greedy", "greedy register allocator", createGreedyVGPRRegisterAllocator);
412
413static VGPRRegisterRegAlloc fastRegAllocVGPR(
414 "fast", "fast register allocator", createFastVGPRRegisterAllocator);
415static WWMRegisterRegAlloc basicRegAllocWWMReg("basic",
416 "basic register allocator",
417 createBasicWWMRegisterAllocator);
418static WWMRegisterRegAlloc
419 greedyRegAllocWWMReg("greedy", "greedy register allocator",
420 createGreedyWWMRegisterAllocator);
421static WWMRegisterRegAlloc fastRegAllocWWMReg("fast", "fast register allocator",
422 createFastWWMRegisterAllocator);
423
424static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {
425 return Phase == ThinOrFullLTOPhase::FullLTOPreLink ||
426 Phase == ThinOrFullLTOPhase::ThinLTOPreLink;
427}
428} // anonymous namespace
429
430static cl::opt<bool, true> EnableObjectLinking(
431 "amdgpu-enable-object-linking",
432 cl::desc("Enable object linking for cross-TU LDS and ABI support"),
433 cl::location(L&: AMDGPUTargetMachine::EnableObjectLinking), cl::init(Val: false),
434 cl::Hidden);
435
436static cl::opt<bool, true> EnableLowerModuleLDS(
437 "amdgpu-enable-lower-module-lds", cl::desc("Enable lower module lds pass"),
438 cl::location(L&: AMDGPUTargetMachine::EnableLowerModuleLDS), cl::init(Val: true),
439 cl::Hidden);
440
441// Scheduler selection is consulted both when creating the scheduler and from
442// overrideSchedPolicy(), so keep the attribute and global command line handling
443// in one helper.
444StringRef llvm::AMDGPU::getSchedStrategy(const AMDGPUOptions &CLOpts,
445 const Function &F) {
446 Attribute SchedStrategyAttr = F.getFnAttribute(Kind: "amdgpu-sched-strategy");
447 if (SchedStrategyAttr.isValid())
448 return SchedStrategyAttr.getValueAsString();
449
450 return CLOpts.sched_strategy;
451}
452
453static void
454diagnoseUnsupportedCoExecSchedulerSelection(const Function &F,
455 const GCNSubtarget &ST) {
456 if (ST.hasGFX1250Insts() || ST.hasGFX950Insts())
457 return;
458
459 F.getContext().diagnose(DI: DiagnosticInfoUnsupported(
460 F,
461 "'amdgpu-sched-strategy'='coexec' is only supported for gfx1250/gfx950",
462 DiagnosticLocation(), DS_Warning));
463}
464
465static bool useNoopPostScheduler(const AMDGPUOptions &CLOpts,
466 const Function &F) {
467 Attribute PostSchedStrategyAttr =
468 F.getFnAttribute(Kind: "amdgpu-post-sched-strategy");
469 if (PostSchedStrategyAttr.isValid())
470 return PostSchedStrategyAttr.getValueAsString() == "nop";
471 return AMDGPU::getSchedStrategy(CLOpts, F) == "coexec";
472}
473
474extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
475 // Register the target
476 RegisterTargetMachine<R600TargetMachine> X(getTheR600Target());
477 RegisterTargetMachine<GCNTargetMachine> Y(getTheGCNTarget());
478 RegisterTargetMachine<GCNTargetMachine> YLegacy(getTheGCNLegacyTarget());
479 static opt::RegisterLibraryOptions<AMDGPUOptions> O;
480
481 PassRegistry *PR = PassRegistry::getPassRegistry();
482 initializeR600ClauseMergePassPass(*PR);
483 initializeR600ControlFlowFinalizerPass(*PR);
484 initializeR600PacketizerPass(*PR);
485 initializeR600ExpandSpecialInstrsPassPass(*PR);
486 initializeR600VectorRegMergerPass(*PR);
487 initializeR600EmitClauseMarkersPass(*PR);
488 initializeR600MachineCFGStructurizerPass(*PR);
489 initializeGlobalISel(*PR);
490 initializeAMDGPUAsmPrinterPass(*PR);
491 initializeAMDGPUDAGToDAGISelLegacyPass(*PR);
492 initializeAMDGPUPrepareAGPRAllocLegacyPass(*PR);
493 initializeGCNDPPCombineLegacyPass(*PR);
494 initializeSILowerI1CopiesLegacyPass(*PR);
495 initializeAMDGPUGlobalISelDivergenceLoweringLegacyPass(*PR);
496 initializeAMDGPURegBankSelectLegacyPass(*PR);
497 initializeAMDGPURegBankLegalizeLegacyPass(*PR);
498 initializeSILowerWWMCopiesLegacyPass(*PR);
499 initializeAMDGPUMarkLastScratchLoadLegacyPass(*PR);
500 initializeSILowerSGPRSpillsLegacyPass(*PR);
501 initializeSIFixSGPRCopiesLegacyPass(*PR);
502 initializeSIFixVGPRCopiesLegacyPass(*PR);
503 initializeSIFoldOperandsLegacyPass(*PR);
504 initializeSIPeepholeSDWALegacyPass(*PR);
505 initializeSIShrinkInstructionsLegacyPass(*PR);
506 initializeSIOptimizeExecMaskingPreRALegacyPass(*PR);
507 initializeSIOptimizeVGPRLiveRangeLegacyPass(*PR);
508 initializeAMDGPUNextUseAnalysisLegacyPassPass(*PR);
509 initializeAMDGPUNextUseAnalysisPrinterLegacyPassPass(*PR);
510 initializeSILoadStoreOptimizerLegacyPass(*PR);
511 initializeAMDGPUCtorDtorLoweringLegacyPass(*PR);
512 initializeAMDGPUAlwaysInlinePass(*PR);
513 initializeAMDGPULowerExecSyncLegacyPass(*PR);
514 initializeAMDGPUSwLowerLDSLegacyPass(*PR);
515 initializeAMDGPUAnnotateUniformValuesLegacyPass(*PR);
516 initializeAMDGPUAtomicOptimizerPass(*PR);
517 initializeAMDGPULowerKernelArgumentsPass(*PR);
518 initializeAMDGPUPromoteKernelArgumentsPass(*PR);
519 initializeAMDGPULowerKernelAttributesPass(*PR);
520 initializeAMDGPUExportKernelRuntimeHandlesLegacyPass(*PR);
521 initializeAMDGPUPostLegalizerCombinerLegacyPass(*PR);
522 initializeAMDGPUPreLegalizerCombinerLegacyPass(*PR);
523 initializeAMDGPURegBankCombinerLegacyPass(*PR);
524 initializeAMDGPUPromoteAllocaPass(*PR);
525 initializeAMDGPUCodeGenPreparePass(*PR);
526 initializeAMDGPULateCodeGenPrepareLegacyPass(*PR);
527 initializeAMDGPURemoveIncompatibleFunctionsLegacyPass(*PR);
528 initializeAMDGPULowerModuleLDSLegacyPass(*PR);
529 initializeAMDGPULowerBufferFatPointersPass(*PR);
530 initializeAMDGPULowerIntrinsicsLegacyPass(*PR);
531 initializeAMDGPUReserveWWMRegsLegacyPass(*PR);
532 initializeAMDGPURewriteAGPRCopyMFMALegacyPass(*PR);
533 initializeAMDGPURewriteOutArgumentsPass(*PR);
534 initializeAMDGPURewriteUndefForPHILegacyPass(*PR);
535 initializeSIAnnotateControlFlowLegacyPass(*PR);
536 initializeAMDGPUInsertDelayAluLegacyPass(*PR);
537 initializeAMDGPULowerVGPREncodingLegacyPass(*PR);
538 initializeSIInsertHardClausesLegacyPass(*PR);
539 initializeSIInsertAsyncBarrierArriveWorkaroundLegacyPass(*PR);
540 initializeSIInsertWaitcntsLegacyPass(*PR);
541 initializeSIModeRegisterLegacyPass(*PR);
542 initializeSIWholeQuadModeLegacyPass(*PR);
543 initializeSILowerControlFlowLegacyPass(*PR);
544 initializeSIPreEmitPeepholeLegacyPass(*PR);
545 initializeSILateBranchLoweringLegacyPass(*PR);
546 initializeSIMemoryLegalizerLegacyPass(*PR);
547 initializeSIOptimizeExecMaskingLegacyPass(*PR);
548 initializeSIPreAllocateWWMRegsLegacyPass(*PR);
549 initializeSIFormMemoryClausesLegacyPass(*PR);
550 initializeSIPostRABundlerLegacyPass(*PR);
551 initializeGCNCreateVOPDLegacyPass(*PR);
552 initializeAMDGPUUnifyDivergentExitNodesLegacyPass(*PR);
553 initializeAMDGPUAAWrapperPassPass(*PR);
554 initializeAMDGPUExternalAAWrapperPass(*PR);
555 initializeAMDGPUImageIntrinsicOptimizerPass(*PR);
556 initializeAMDGPUPrintfRuntimeBindingPass(*PR);
557 initializeAMDGPUResourceUsageAnalysisWrapperPassPass(*PR);
558 initializeGCNNSAReassignLegacyPass(*PR);
559 initializeGCNPreRAOptimizationsLegacyPass(*PR);
560 initializeGCNPreRALongBranchRegLegacyPass(*PR);
561 initializeGCNRewritePartialRegUsesLegacyPass(*PR);
562 initializeGCNRegPressurePrinterPass(*PR);
563 initializeAMDGPUPreloadKernArgPrologLegacyPass(*PR);
564 initializeAMDGPUWaitSGPRHazardsLegacyPass(*PR);
565 initializeAMDGPUPreloadKernelArgumentsLegacyPass(*PR);
566 initializeAMDGPUUniformIntrinsicCombineLegacyPass(*PR);
567 initializeSIPostRA16BitMovFoldingLegacyPass(*PR);
568}
569
570static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
571 return std::make_unique<AMDGPUTargetObjectFile>();
572}
573
574static ScheduleDAGInstrs *createSIMachineScheduler(MachineSchedContext *C) {
575 return new SIScheduleDAGMI(C);
576}
577
578static ScheduleDAGInstrs *
579createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
580 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
581 ScheduleDAGMILive *DAG =
582 new GCNScheduleDAGMILive(C, std::make_unique<GCNMaxOccupancySchedStrategy>(args&: C));
583 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII));
584 if (ST.shouldClusterStores())
585 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII));
586 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
587 DAG->addMutation(Mutation: createAMDGPUMacroFusionDAGMutation());
588 DAG->addMutation(Mutation: createAMDGPUExportClusteringDAGMutation());
589 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
590 DAG->addMutation(Mutation: createAMDGPUHazardLatencyDAGMutation(MF: C->MF));
591 return DAG;
592}
593
594static ScheduleDAGInstrs *
595createGCNMaxILPMachineScheduler(MachineSchedContext *C) {
596 ScheduleDAGMILive *DAG =
597 new GCNScheduleDAGMILive(C, std::make_unique<GCNMaxILPSchedStrategy>(args&: C));
598 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
599 return DAG;
600}
601
602static ScheduleDAGInstrs *
603createGCNMaxMemoryClauseMachineScheduler(MachineSchedContext *C) {
604 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
605 ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive(
606 C, std::make_unique<GCNMaxMemoryClauseSchedStrategy>(args&: C));
607 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII));
608 if (ST.shouldClusterStores())
609 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII));
610 DAG->addMutation(Mutation: createAMDGPUExportClusteringDAGMutation());
611 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
612 DAG->addMutation(Mutation: createAMDGPUHazardLatencyDAGMutation(MF: C->MF));
613 return DAG;
614}
615
616static ScheduleDAGInstrs *
617createIterativeGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
618 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
619 auto *DAG = new GCNIterativeScheduler(
620 C, GCNIterativeScheduler::SCHEDULE_LEGACYMAXOCCUPANCY);
621 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII));
622 if (ST.shouldClusterStores())
623 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII));
624 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
625 return DAG;
626}
627
628static ScheduleDAGInstrs *createMinRegScheduler(MachineSchedContext *C) {
629 auto *DAG = new GCNIterativeScheduler(
630 C, GCNIterativeScheduler::SCHEDULE_MINREGFORCED);
631 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
632 return DAG;
633}
634
635static ScheduleDAGInstrs *
636createIterativeILPMachineScheduler(MachineSchedContext *C) {
637 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
638 auto *DAG = new GCNIterativeScheduler(C, GCNIterativeScheduler::SCHEDULE_ILP);
639 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII));
640 if (ST.shouldClusterStores())
641 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII));
642 DAG->addMutation(Mutation: createAMDGPUMacroFusionDAGMutation());
643 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
644 return DAG;
645}
646
647static MachineSchedRegistry
648SISchedRegistry("si", "Run SI's custom scheduler",
649 createSIMachineScheduler);
650
651static MachineSchedRegistry
652GCNMaxOccupancySchedRegistry("gcn-max-occupancy",
653 "Run GCN scheduler to maximize occupancy",
654 createGCNMaxOccupancyMachineScheduler);
655
656static MachineSchedRegistry
657 GCNMaxILPSchedRegistry("gcn-max-ilp", "Run GCN scheduler to maximize ilp",
658 createGCNMaxILPMachineScheduler);
659
660static MachineSchedRegistry GCNMaxMemoryClauseSchedRegistry(
661 "gcn-max-memory-clause", "Run GCN scheduler to maximize memory clause",
662 createGCNMaxMemoryClauseMachineScheduler);
663
664static MachineSchedRegistry IterativeGCNMaxOccupancySchedRegistry(
665 "gcn-iterative-max-occupancy-experimental",
666 "Run GCN scheduler to maximize occupancy (experimental)",
667 createIterativeGCNMaxOccupancyMachineScheduler);
668
669static MachineSchedRegistry GCNMinRegSchedRegistry(
670 "gcn-iterative-minreg",
671 "Run GCN iterative scheduler for minimal register usage (experimental)",
672 createMinRegScheduler);
673
674static MachineSchedRegistry GCNILPSchedRegistry(
675 "gcn-iterative-ilp",
676 "Run GCN iterative scheduler for ILP scheduling (experimental)",
677 createIterativeILPMachineScheduler);
678
679LLVM_READNONE
680static StringRef getGPUOrDefault(const Triple &TT, StringRef GPU) {
681 if (!GPU.empty())
682 return GPU;
683
684 if (StringRef Name = AMDGPU::getArchNameFromSubArch(SubArch: TT.getSubArch());
685 !Name.empty())
686 return Name;
687
688 // Need to default to a target with flat support for HSA.
689 if (TT.isAMDGCN())
690 return TT.getOS() == Triple::AMDHSA ? "generic-hsa" : "generic";
691
692 return "r600";
693}
694
695static Reloc::Model getEffectiveRelocModel() {
696 // The AMDGPU toolchain only supports generating shared objects, so we
697 // must always use PIC.
698 return Reloc::PIC_;
699}
700
701AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT,
702 StringRef CPU, StringRef FS,
703 const TargetOptions &Options,
704 std::optional<Reloc::Model> RM,
705 std::optional<CodeModel::Model> CM,
706 CodeGenOptLevel OptLevel)
707 : CodeGenTargetMachineImpl(T, TT, getGPUOrDefault(TT, GPU: CPU), FS, Options,
708 getEffectiveRelocModel(),
709 getEffectiveCodeModel(CM, Default: CodeModel::Small),
710 OptLevel),
711 CLOpts(cl::LibraryOptionsContext::get<AMDGPUOptions>(
712 C: Options.MCOptions.LibraryOptions)),
713 TLOF(createTLOF(TT: getTargetTriple())) {
714 initAsmInfo();
715 if (TT.isAMDGCN()) {
716 // Triple is missing a representation for non-empty, but unrecognized
717 // subarches. Only permit no subarch for any subtarget if it was really
718 // empty.
719 bool IsUnknownSubArch =
720 TT.getSubArch() == Triple::NoSubArch && TT.getArchName().size() != 6;
721 if (IsUnknownSubArch)
722 reportFatalUsageError(reason: "unknown subarch " + TT.getArchName());
723
724 if (TT.getSubArch() != Triple::NoSubArch) {
725 AMDGPU::GPUKind Kind = AMDGPU::parseArchAMDGCN(CPU);
726 Triple::SubArchType GPUSubArch = AMDGPU::getSubArch(AK: Kind);
727 if (Kind != AMDGPU::GK_NONE && GPUSubArch != TT.getSubArch() &&
728 TT.getSubArch() != AMDGPU::getMajorSubArch(SubArch: GPUSubArch)) {
729 reportFatalUsageError(reason: "invalid cpu '" + CPU + "' for subarch " +
730 TT.getArchName());
731 }
732 }
733
734 if (getMCSubtargetInfo().checkFeatures(FS: "+wavefrontsize64"))
735 MRI.reset(p: llvm::createGCNMCRegisterInfo(DwarfFlavour: AMDGPUDwarfFlavour::Wave64));
736 else if (getMCSubtargetInfo().checkFeatures(FS: "+wavefrontsize32"))
737 MRI.reset(p: llvm::createGCNMCRegisterInfo(DwarfFlavour: AMDGPUDwarfFlavour::Wave32));
738 }
739 LLT::setUseExtended(true);
740}
741
742bool AMDGPUTargetMachine::EnableFunctionCalls = false;
743bool AMDGPUTargetMachine::EnableObjectLinking = false;
744bool AMDGPUTargetMachine::EnableLowerModuleLDS = true;
745
746AMDGPUTargetMachine::~AMDGPUTargetMachine() = default;
747
748llvm::ScheduleDAGInstrs *
749AMDGPUTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
750 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
751 ScheduleDAGMILive *DAG = createSchedLive(C);
752 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII));
753 if (ST.shouldClusterStores())
754 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII));
755 return DAG;
756}
757
758/// Predicate for Internalize pass.
759static bool mustPreserveGV(const GlobalValue &GV) {
760 if (const Function *F = dyn_cast<Function>(Val: &GV))
761 return F->isDeclaration() || F->getName().starts_with(Prefix: "__asan_") ||
762 F->getName().starts_with(Prefix: "__sanitizer_") ||
763 AMDGPU::isEntryFunctionCC(CC: F->getCallingConv());
764
765 GV.removeDeadConstantUsers();
766 return !GV.use_empty();
767}
768
769void AMDGPUTargetMachine::registerDefaultAliasAnalyses(AAManager &AAM) {
770 if (CLOpts.enable_aa)
771 AAM.registerFunctionAnalysis<AMDGPUAA>();
772}
773
774static Expected<AMDGPU::ScanOptions>
775parseAMDGPUAtomicOptimizerStrategy(StringRef Params) {
776 if (Params.empty())
777 return AMDGPU::ScanOptions::Iterative;
778 Params.consume_front(Prefix: "strategy=");
779 auto Result = StringSwitch<std::optional<AMDGPU::ScanOptions>>(Params)
780 .Case(S: "dpp", Value: AMDGPU::ScanOptions::DPP)
781 .Cases(CaseStrings: {"iterative", ""}, Value: AMDGPU::ScanOptions::Iterative)
782 .Case(S: "none", Value: AMDGPU::ScanOptions::None)
783 .Default(Value: std::nullopt);
784 if (Result)
785 return *Result;
786 return make_error<StringError>(Args: "invalid parameter", Args: inconvertibleErrorCode());
787}
788
789Expected<AMDGPUAttributorOptions>
790parseAMDGPUAttributorPassOptions(StringRef Params) {
791 AMDGPUAttributorOptions Result;
792 while (!Params.empty()) {
793 StringRef ParamName;
794 std::tie(args&: ParamName, args&: Params) = Params.split(Separator: ';');
795 if (ParamName == "closed-world") {
796 Result.IsClosedWorld = true;
797 } else {
798 return make_error<StringError>(
799 Args: formatv(Fmt: "invalid AMDGPUAttributor pass parameter '{0}' ", Vals&: ParamName)
800 .str(),
801 Args: inconvertibleErrorCode());
802 }
803 }
804 return Result;
805}
806
807void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
808
809#define GET_PASS_REGISTRY "AMDGPUPassRegistry.def"
810#include "llvm/Passes/TargetPassRegistry.inc"
811
812 PB.registerPipelineParsingCallback(
813 C: [this](StringRef Name, CGSCCPassManager &PM,
814 ArrayRef<PassBuilder::PipelineElement> Pipeline) {
815 if (Name == "amdgpu-attributor-cgscc" && getTargetTriple().isAMDGCN()) {
816 PM.addPass(Pass: AMDGPUAttributorCGSCCPass(
817 *static_cast<GCNTargetMachine *>(this)));
818 return true;
819 }
820 return false;
821 });
822
823 PB.registerScalarOptimizerLateEPCallback(
824 C: [](FunctionPassManager &FPM, OptimizationLevel Level) {
825 if (Level == OptimizationLevel::O0)
826 return;
827
828 FPM.addPass(Pass: InferAddressSpacesPass());
829 });
830
831 PB.registerVectorizerEndEPCallback(
832 C: [](FunctionPassManager &FPM, OptimizationLevel Level) {
833 if (Level == OptimizationLevel::O0)
834 return;
835
836 FPM.addPass(Pass: InferAddressSpacesPass());
837 });
838
839 PB.registerPipelineEarlySimplificationEPCallback(
840 C: [this](ModulePassManager &PM, OptimizationLevel Level,
841 ThinOrFullLTOPhase Phase) {
842 if (!isLTOPreLink(Phase) && getTargetTriple().isAMDGCN()) {
843 // When we are not using -fgpu-rdc, we can run accelerator code
844 // selection relatively early, but still after linking to prevent
845 // eager removal of potentially reachable symbols.
846 if (CLOpts.enable_hipstdpar) {
847 PM.addPass(Pass: HipStdParMathFixupPass());
848 PM.addPass(Pass: HipStdParAcceleratorCodeSelectionPass());
849 }
850
851 PM.addPass(Pass: AMDGPUPrintfRuntimeBindingPass());
852 }
853
854 if (Level == OptimizationLevel::O0)
855 return;
856
857 // We don't want to run internalization at per-module stage.
858 if (CLOpts.internalize_symbols && !isLTOPreLink(Phase)) {
859 PM.addPass(Pass: InternalizePass(mustPreserveGV));
860 PM.addPass(Pass: GlobalDCEPass());
861 }
862
863 if (CLOpts.early_inline_all && !EnableFunctionCalls)
864 PM.addPass(Pass: AMDGPUAlwaysInlinePass());
865 });
866
867 PB.registerPeepholeEPCallback(
868 C: [this](FunctionPassManager &FPM, OptimizationLevel Level) {
869 if (Level == OptimizationLevel::O0)
870 return;
871
872 FPM.addPass(Pass: AMDGPUUseNativeCallsPass());
873 if (CLOpts.simplify_libcall)
874 FPM.addPass(Pass: AMDGPUSimplifyLibCallsPass());
875
876 if (CLOpts.enable_uniform_intrinsic_combine)
877 FPM.addPass(Pass: AMDGPUUniformIntrinsicCombinePass());
878 });
879
880 PB.registerCGSCCOptimizerLateEPCallback(
881 C: [this](CGSCCPassManager &PM, OptimizationLevel Level) {
882 if (Level == OptimizationLevel::O0)
883 return;
884
885 FunctionPassManager FPM;
886
887 // Add promote kernel arguments pass to the opt pipeline right before
888 // infer address spaces which is needed to do actual address space
889 // rewriting.
890 if (Level > OptimizationLevel::O1 &&
891 CLOpts.enable_promote_kernel_arguments)
892 FPM.addPass(Pass: AMDGPUPromoteKernelArgumentsPass());
893
894 // Add infer address spaces pass to the opt pipeline after inlining
895 // but before SROA to increase SROA opportunities.
896 FPM.addPass(Pass: InferAddressSpacesPass());
897
898 // This should run after inlining to have any chance of doing
899 // anything, and before other cleanup optimizations.
900 FPM.addPass(Pass: AMDGPULowerKernelAttributesPass());
901
902 // Promote alloca to vector before SROA and loop unroll. If we
903 // manage to eliminate allocas before unroll we may choose to unroll
904 // less.
905 FPM.addPass(Pass: AMDGPUPromoteAllocaToVectorPass(*this));
906
907 PM.addPass(Pass: createCGSCCToFunctionPassAdaptor(Pass: std::move(FPM)));
908 });
909
910 // FIXME: Why is AMDGPUAttributor not in CGSCC?
911 PB.registerOptimizerLastEPCallback(C: [this](ModulePassManager &MPM,
912 OptimizationLevel Level,
913 ThinOrFullLTOPhase Phase) {
914 if (Level != OptimizationLevel::O0) {
915 if (!isLTOPreLink(Phase)) {
916 if (CLOpts.attributor_enable && getTargetTriple().isAMDGCN()) {
917 AMDGPUAttributorOptions Opts;
918 MPM.addPass(Pass: AMDGPUAttributorPass(*this, Opts, Phase));
919 }
920 }
921 }
922 });
923
924 PB.registerFullLinkTimeOptimizationLastEPCallback(
925 C: [this](ModulePassManager &PM, OptimizationLevel Level) {
926 // Clean up redundant memory round-trips that the full-LTO pipeline,
927 // unlike the non-LTO/ThinLTO ones, otherwise leaves for codegen.
928 if (Level != OptimizationLevel::O0) {
929 PM.addPass(Pass: createModuleToFunctionPassAdaptor(
930 Pass: EarlyCSEPass(/*UseMemorySSA=*/true)));
931 }
932
933 // When we are using -fgpu-rdc, we can only run accelerator code
934 // selection after linking to prevent, otherwise we end up removing
935 // potentially reachable symbols that were exported as external in other
936 // modules.
937 if (CLOpts.enable_hipstdpar) {
938 PM.addPass(Pass: HipStdParMathFixupPass());
939 PM.addPass(Pass: HipStdParAcceleratorCodeSelectionPass());
940 }
941 // We want to support the -lto-partitions=N option as "best effort".
942 // For that, we need to lower LDS earlier in the pipeline before the
943 // module is partitioned for codegen.
944 if (CLOpts.enable_lower_exec_sync)
945 PM.addPass(Pass: AMDGPULowerExecSyncPass());
946 if (CLOpts.enable_sw_lower_lds)
947 PM.addPass(Pass: AMDGPUSwLowerLDSPass());
948 if (EnableLowerModuleLDS)
949 PM.addPass(Pass: AMDGPULowerModuleLDSPass(*this));
950 if (Level != OptimizationLevel::O0) {
951 // We only want to run this with O2 or higher since inliner and SROA
952 // don't run in O1.
953 if (Level != OptimizationLevel::O1) {
954 PM.addPass(
955 Pass: createModuleToFunctionPassAdaptor(Pass: InferAddressSpacesPass()));
956 }
957 // Do we really need internalization in LTO?
958 if (CLOpts.internalize_symbols) {
959 PM.addPass(Pass: InternalizePass(mustPreserveGV));
960 PM.addPass(Pass: GlobalDCEPass());
961 }
962 if (CLOpts.attributor_enable && getTargetTriple().isAMDGCN()) {
963 AMDGPUAttributorOptions Opt;
964 if (CLOpts.link_time_closed_world)
965 Opt.IsClosedWorld = true;
966 PM.addPass(Pass: AMDGPUAttributorPass(
967 *this, Opt, ThinOrFullLTOPhase::FullLTOPostLink));
968 }
969 }
970 if (!NoKernelInfoEndLTO) {
971 FunctionPassManager FPM;
972 FPM.addPass(Pass: KernelInfoPrinter(this));
973 PM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
974 }
975 });
976
977 PB.registerRegClassFilterParsingCallback(
978 C: [](StringRef FilterName) -> RegAllocFilterFunc {
979 if (FilterName == "sgpr")
980 return onlyAllocateSGPRs;
981 if (FilterName == "vgpr")
982 return onlyAllocateVGPRs;
983 if (FilterName == "wwm")
984 return onlyAllocateWWMRegs;
985 return nullptr;
986 });
987}
988
989bool AMDGPUTargetMachine::isNoopAddrSpaceCast(const DataLayout &DL,
990 unsigned SrcAS,
991 unsigned DestAS) const {
992 return AMDGPU::isFlatGlobalAddrSpace(AS: SrcAS) &&
993 AMDGPU::isFlatGlobalAddrSpace(AS: DestAS);
994}
995
996unsigned AMDGPUTargetMachine::getAssumedAddrSpace(const Value *V) const {
997 if (auto *Arg = dyn_cast<Argument>(Val: V);
998 Arg &&
999 AMDGPU::isModuleEntryFunctionCC(CC: Arg->getParent()->getCallingConv()) &&
1000 !Arg->hasByRefAttr())
1001 return AMDGPUAS::GLOBAL_ADDRESS;
1002
1003 const auto *LD = dyn_cast<LoadInst>(Val: V);
1004 if (!LD) // TODO: Handle invariant load like constant.
1005 return AMDGPUAS::UNKNOWN_ADDRESS_SPACE;
1006
1007 // It must be a generic pointer loaded.
1008 assert(V->getType()->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS);
1009
1010 const auto *Ptr = LD->getPointerOperand();
1011 if (Ptr->getType()->getPointerAddressSpace() != AMDGPUAS::CONSTANT_ADDRESS)
1012 return AMDGPUAS::UNKNOWN_ADDRESS_SPACE;
1013 // For a generic pointer loaded from the constant memory, it could be assumed
1014 // as a global pointer since the constant memory is only populated on the
1015 // host side. As implied by the offload programming model, only global
1016 // pointers could be referenced on the host side.
1017 return AMDGPUAS::GLOBAL_ADDRESS;
1018}
1019
1020std::pair<const Value *, unsigned>
1021AMDGPUTargetMachine::getPredicatedAddrSpace(const Value *V) const {
1022 if (auto *II = dyn_cast<IntrinsicInst>(Val: V)) {
1023 switch (II->getIntrinsicID()) {
1024 case Intrinsic::amdgcn_is_shared:
1025 return std::pair(II->getArgOperand(i: 0), AMDGPUAS::LOCAL_ADDRESS);
1026 case Intrinsic::amdgcn_is_private:
1027 return std::pair(II->getArgOperand(i: 0), AMDGPUAS::PRIVATE_ADDRESS);
1028 default:
1029 break;
1030 }
1031 return std::pair(nullptr, -1);
1032 }
1033 // Check the global pointer predication based on
1034 // (!is_share(p) && !is_private(p)). Note that logic 'and' is commutative and
1035 // the order of 'is_shared' and 'is_private' is not significant.
1036 Value *Ptr;
1037 if (match(
1038 V: const_cast<Value *>(V),
1039 P: m_c_And(L: m_Not(V: m_Intrinsic<Intrinsic::amdgcn_is_shared>(Ops: m_Value(V&: Ptr))),
1040 R: m_Not(V: m_Intrinsic<Intrinsic::amdgcn_is_private>(
1041 Ops: m_Deferred(V: Ptr))))))
1042 return std::pair(Ptr, AMDGPUAS::GLOBAL_ADDRESS);
1043
1044 return std::pair(nullptr, -1);
1045}
1046
1047unsigned
1048AMDGPUTargetMachine::getAddressSpaceForPseudoSourceKind(unsigned Kind) const {
1049 switch (Kind) {
1050 case PseudoSourceValue::Stack:
1051 case PseudoSourceValue::FixedStack:
1052 return AMDGPUAS::PRIVATE_ADDRESS;
1053 case PseudoSourceValue::ConstantPool:
1054 case PseudoSourceValue::GOT:
1055 case PseudoSourceValue::JumpTable:
1056 case PseudoSourceValue::GlobalValueCallEntry:
1057 case PseudoSourceValue::ExternalSymbolCallEntry:
1058 return AMDGPUAS::CONSTANT_ADDRESS;
1059 }
1060 return AMDGPUAS::FLAT_ADDRESS;
1061}
1062
1063bool AMDGPUTargetMachine::splitModule(
1064 Module &M, unsigned NumParts,
1065 function_ref<void(std::unique_ptr<Module> MPart)> ModuleCallback) {
1066 // FIXME(?): Would be better to use an already existing Analysis/PassManager,
1067 // but all current users of this API don't have one ready and would need to
1068 // create one anyway. Let's hide the boilerplate for now to keep it simple.
1069
1070 LoopAnalysisManager LAM;
1071 FunctionAnalysisManager FAM;
1072 CGSCCAnalysisManager CGAM;
1073 ModuleAnalysisManager MAM;
1074
1075 PassBuilder PB(M.getContext(), this);
1076 PB.registerModuleAnalyses(MAM);
1077 PB.registerFunctionAnalyses(FAM);
1078 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
1079
1080 ModulePassManager MPM;
1081 MPM.addPass(Pass: AMDGPUSplitModulePass(NumParts, ModuleCallback));
1082 MPM.run(IR&: M, AM&: MAM);
1083 return true;
1084}
1085
1086//===----------------------------------------------------------------------===//
1087// GCN Target Machine (SI+)
1088//===----------------------------------------------------------------------===//
1089
1090GCNTargetMachine::GCNTargetMachine(const Target &T, const Triple &TT,
1091 StringRef CPU, StringRef FS,
1092 const TargetOptions &Options,
1093 std::optional<Reloc::Model> RM,
1094 std::optional<CodeModel::Model> CM,
1095 CodeGenOptLevel OL, bool JIT)
1096 : AMDGPUTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {
1097 setEnableDefaultMachineVerifier(false);
1098 // addFastRegAlloc inserts SIWholeQuadMode after TwoAddressInstructionPass.
1099 setEnableTiedFastRegAlloc(false);
1100}
1101
1102enum class OOBFlagValue {
1103 Any = 0,
1104 Relaxed = 1,
1105 Strict = 2,
1106};
1107
1108/// Returns the OOB mode encoded by a module flag.
1109/// An absent flag defaults to Any.
1110static OOBFlagValue getOOBFlagValue(const Module &M, StringRef FlagName) {
1111 const auto *Flag =
1112 mdconst::dyn_extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: FlagName));
1113 if (!Flag)
1114 return OOBFlagValue::Any;
1115 return static_cast<OOBFlagValue>(Flag->getZExtValue());
1116}
1117
1118/// Returns the xnack/sramecc setting encoded by a module flag.
1119/// Module flag values: 0 = disabled, 1 = enabled.
1120/// An absent flag defaults to Any.
1121AMDGPU::TargetIDSetting
1122GCNTargetMachine::getTargetIDSettingFromModuleFlag(const Module &M,
1123 StringRef FlagName) const {
1124 using AMDGPU::TargetIDSetting;
1125
1126 if (CLOpts.xnack != BoolOrDefault::Default && FlagName == "amdgpu.xnack")
1127 return CLOpts.xnack == BoolOrDefault::True ? TargetIDSetting::On
1128 : TargetIDSetting::Off;
1129 if (CLOpts.sramecc != BoolOrDefault::Default && FlagName == "amdgpu.sramecc")
1130 return CLOpts.sramecc == BoolOrDefault::True ? TargetIDSetting::On
1131 : TargetIDSetting::Off;
1132
1133 const auto *Flag =
1134 mdconst::dyn_extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: FlagName));
1135 if (!Flag)
1136 return TargetIDSetting::Any;
1137 return Flag->getZExtValue() == 0 ? TargetIDSetting::Off : TargetIDSetting::On;
1138}
1139
1140const TargetSubtargetInfo *
1141GCNTargetMachine::getSubtargetImpl(const Function &F) const {
1142 AttributeSet FnAttrs = F.getAttributes().getFnAttrs();
1143 const Module *M = F.getParent();
1144 if (const GCNSubtarget *ST = LastSubtarget.lookup(NewFnAttrs: FnAttrs, NewM: M))
1145 return ST;
1146
1147 SubtargetAttrs Attrs;
1148 Attrs.GPU = FnAttrs.getAttribute(Kind: "target-cpu");
1149 Attrs.FS = FnAttrs.getAttribute(Kind: "target-features");
1150 Attrs.BufRelaxed =
1151 getOOBFlagValue(M: *M, FlagName: AMDGPUOOBMode::BufferFlag) == OOBFlagValue::Relaxed;
1152 Attrs.TBufRelaxed =
1153 getOOBFlagValue(M: *M, FlagName: AMDGPUOOBMode::TBufferFlag) == OOBFlagValue::Relaxed;
1154 Attrs.Xnack = getTargetIDSettingFromModuleFlag(M: *M, FlagName: "amdgpu.xnack");
1155 Attrs.SramEcc = getTargetIDSettingFromModuleFlag(M: *M, FlagName: "amdgpu.sramecc");
1156
1157 if (const GCNSubtarget *ST = LastSubtarget.lookup(NewFnAttrs: FnAttrs, NewM: M, NewAttrs: Attrs))
1158 return ST;
1159
1160 StringRef GPU =
1161 Attrs.GPU.isValid() ? Attrs.GPU.getValueAsString() : getTargetCPU();
1162 StringRef FS = Attrs.FS.isValid() ? Attrs.FS.getValueAsString()
1163 : getTargetFeatureString();
1164
1165 using AMDGPU::TargetIDSetting;
1166 SmallString<128> SubtargetKey(GPU);
1167 SubtargetKey.append(RHS: FS);
1168 if (Attrs.BufRelaxed)
1169 SubtargetKey.append(RHS: ",buf-oob=1");
1170 if (Attrs.TBufRelaxed)
1171 SubtargetKey.append(RHS: ",tbuf-oob=1");
1172 if (Attrs.Xnack != TargetIDSetting::Any) {
1173 SubtargetKey.append(RHS: ",xnack=");
1174 SubtargetKey.push_back(Elt: Attrs.Xnack == TargetIDSetting::On ? '1' : '0');
1175 }
1176 if (Attrs.SramEcc != TargetIDSetting::Any) {
1177 SubtargetKey.append(RHS: ",sramecc=");
1178 SubtargetKey.push_back(Elt: Attrs.SramEcc == TargetIDSetting::On ? '1' : '0');
1179 }
1180
1181 auto &I = SubtargetMap[SubtargetKey];
1182 if (!I) {
1183 AMDGPU::GPUKind Kind = AMDGPU::parseArchAMDGCN(CPU: GPU);
1184 Triple::SubArchType GPUSubArch = AMDGPU::getSubArch(AK: Kind);
1185
1186 // Enforce the subtarget is covered by the subarch. Tolerate no subarch for
1187 // legacy compatibility.
1188 const Triple &TT = M->getTargetTriple();
1189 if (GPUSubArch != TT.getSubArch() && Kind != AMDGPU::GK_NONE) {
1190 // Check if this is a generic subarch which has subtargets. Ignore
1191 // unknown subtargets with a known subarch, since for whatever reason
1192 // the convention is to just print a warning and ignore unrecognized
1193 // subtargets.
1194 bool IsLegacyEmptySubArch = TT.getSubArch() == Triple::NoSubArch;
1195 if (!IsLegacyEmptySubArch &&
1196 AMDGPU::getMajorSubArch(SubArch: GPUSubArch) != TT.getSubArch()) {
1197 F.getContext().emitError(ErrorStr: "invalid subtarget '" + Twine(GPU) +
1198 "' for subarch " + TT.getArchName());
1199 }
1200 }
1201
1202 I = std::make_unique<GCNSubtarget>(args: TargetTriple, args&: GPU, args&: FS, args: *this,
1203 args&: Attrs.BufRelaxed, args&: Attrs.TBufRelaxed,
1204 args&: Attrs.Xnack, args&: Attrs.SramEcc);
1205 }
1206
1207 LastSubtarget.set(NewFnAttrs: FnAttrs, NewM: M, NewAttrs: Attrs, NewSubtarget: I.get());
1208 return I.get();
1209}
1210
1211TargetTransformInfo
1212GCNTargetMachine::getTargetTransformInfo(const Function &F) const {
1213 return TargetTransformInfo(std::make_unique<GCNTTIImpl>(args: this, args: F));
1214}
1215
1216Error GCNTargetMachine::buildCodeGenPipeline(
1217 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
1218 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
1219 const CGPassBuilderOption &Opts, MCContext &Ctx,
1220 PassInstrumentationCallbacks *PIC) {
1221 AMDGPUCodeGenPassBuilder CGPB(*this, Opts, PIC);
1222 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
1223}
1224
1225ScheduleDAGInstrs *
1226GCNTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
1227 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
1228 if (ST.enableSIScheduler())
1229 return createSIMachineScheduler(C);
1230
1231 StringRef SchedStrategy =
1232 AMDGPU::getSchedStrategy(CLOpts, F: C->MF->getFunction());
1233
1234 if (SchedStrategy == "max-ilp")
1235 return createGCNMaxILPMachineScheduler(C);
1236
1237 if (SchedStrategy == "max-memory-clause")
1238 return createGCNMaxMemoryClauseMachineScheduler(C);
1239
1240 if (SchedStrategy == "iterative-ilp")
1241 return createIterativeILPMachineScheduler(C);
1242
1243 if (SchedStrategy == "iterative-minreg")
1244 return createMinRegScheduler(C);
1245
1246 if (SchedStrategy == "iterative-maxocc")
1247 return createIterativeGCNMaxOccupancyMachineScheduler(C);
1248
1249 if (SchedStrategy == "coexec") {
1250 diagnoseUnsupportedCoExecSchedulerSelection(F: C->MF->getFunction(), ST);
1251 return createGCNCoExecMachineScheduler(C);
1252 }
1253
1254 return createGCNMaxOccupancyMachineScheduler(C);
1255}
1256
1257ScheduleDAGInstrs *
1258GCNTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
1259 if (useNoopPostScheduler(CLOpts, F: C->MF->getFunction()))
1260 return createGCNNoopPostMachineScheduler(C);
1261
1262 ScheduleDAGMI *DAG =
1263 new GCNPostScheduleDAGMILive(C, std::make_unique<PostGenericScheduler>(args&: C),
1264 /*RemoveKillFlags=*/true);
1265 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
1266 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII));
1267 if (ST.shouldClusterStores())
1268 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII));
1269 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::PostRA));
1270 if (valueOr(X: CLOpts.enable_vopd, Default: getOptLevel() >= CodeGenOptLevel::Less))
1271 DAG->addMutation(Mutation: createVOPDPairingMutation());
1272 DAG->addMutation(Mutation: createAMDGPUExportClusteringDAGMutation());
1273 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
1274 DAG->addMutation(Mutation: createAMDGPUHazardLatencyDAGMutation(MF: C->MF));
1275 return DAG;
1276}
1277//===----------------------------------------------------------------------===//
1278// AMDGPU Legacy Pass Setup
1279//===----------------------------------------------------------------------===//
1280
1281std::unique_ptr<CSEConfigBase> llvm::AMDGPUPassConfig::getCSEConfig() const {
1282 return getStandardCSEConfigForOpt(Level: TM->getOptLevel());
1283}
1284
1285namespace {
1286
1287class GCNPassConfig final : public AMDGPUPassConfig {
1288public:
1289 GCNPassConfig(TargetMachine &TM, PassManagerBase &PM)
1290 : AMDGPUPassConfig(TM, PM) {
1291 substitutePass(StandardID: &PostRASchedulerID, TargetID: &PostMachineSchedulerID);
1292 }
1293
1294 bool addPreISel() override;
1295 void addMachineSSAOptimization() override;
1296 bool addILPOpts() override;
1297 bool addInstSelector() override;
1298 bool addIRTranslator() override;
1299 void addPreLegalizeMachineIR() override;
1300 bool addLegalizeMachineIR() override;
1301 void addPreRegBankSelect() override;
1302 bool addRegBankSelect() override;
1303 void addPreGlobalInstructionSelect() override;
1304 bool addGlobalInstructionSelect() override;
1305 void addPreRegAlloc() override;
1306 void addFastRegAlloc() override;
1307 void addOptimizedRegAlloc() override;
1308
1309 FunctionPass *createSGPRAllocPass(bool Optimized);
1310 FunctionPass *createVGPRAllocPass(bool Optimized);
1311 FunctionPass *createWWMRegAllocPass(bool Optimized);
1312 FunctionPass *createRegAllocPass(bool Optimized) override;
1313
1314 bool addRegAssignAndRewriteFast() override;
1315 bool addRegAssignAndRewriteOptimized() override;
1316
1317 bool addPreRewrite() override;
1318 void addPostRegAlloc() override;
1319 void addPreSched2() override;
1320 void addPreEmitPass() override;
1321 void addPostBBSections() override;
1322};
1323
1324} // end anonymous namespace
1325
1326AMDGPUPassConfig::AMDGPUPassConfig(TargetMachine &TM, PassManagerBase &PM)
1327 : TargetPassConfig(TM, PM), CLOpts(getAMDGPUTargetMachine().getCLOpts()) {
1328 // Exceptions and StackMaps are not supported, so these passes will never do
1329 // anything.
1330 disablePass(PassID: &StackMapLivenessID);
1331 disablePass(PassID: &FuncletLayoutID);
1332 // Garbage collection is not supported.
1333 disablePass(PassID: &GCLoweringID);
1334 disablePass(PassID: &ShadowStackGCLoweringID);
1335
1336 if (CLOpts.use_ssa_machine_scheduler) {
1337 // Use SSA Machine Scheduler instead of regular Machine Scheduler.
1338 disablePass(PassID: &MachineSchedulerID);
1339 setEnableSSAMachineScheduler(true);
1340 }
1341}
1342
1343void AMDGPUPassConfig::addEarlyCSEOrGVNPass() {
1344 if (getOptLevel() == CodeGenOptLevel::Aggressive)
1345 addPass(P: createGVNPass());
1346 else
1347 addPass(P: createEarlyCSEPass());
1348}
1349
1350void AMDGPUPassConfig::addStraightLineScalarOptimizationPasses() {
1351 if (CLOpts.loop_prefetch)
1352 addPass(P: createLoopDataPrefetchPass());
1353 addPass(P: createSeparateConstOffsetFromGEPPass());
1354 // ReassociateGEPs exposes more opportunities for SLSR. See
1355 // the example in reassociate-geps-and-slsr.ll.
1356 addPass(P: createStraightLineStrengthReducePass());
1357 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or
1358 // EarlyCSE can reuse.
1359 addEarlyCSEOrGVNPass();
1360 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
1361 addPass(P: createNaryReassociatePass());
1362 // NaryReassociate on GEPs creates redundant common expressions, so run
1363 // EarlyCSE after it.
1364 addPass(P: createEarlyCSEPass());
1365}
1366
1367void AMDGPUPassConfig::addIRPasses() {
1368 const AMDGPUTargetMachine &TM = getAMDGPUTargetMachine();
1369
1370 if (CLOpts.enable_remove_incompatible_functions &&
1371 TM.getTargetTriple().isAMDGCN())
1372 addPass(P: createAMDGPURemoveIncompatibleFunctionsPass(&TM));
1373
1374 // There is no reason to run these.
1375 disablePass(PassID: &StackMapLivenessID);
1376 disablePass(PassID: &FuncletLayoutID);
1377 disablePass(PassID: &PatchableFunctionID);
1378
1379 if (TM.getTargetTriple().isAMDGCN())
1380 addPass(P: createAMDGPUPrintfRuntimeBinding());
1381
1382 if (CLOpts.lower_global_ctor_dtor)
1383 addPass(P: createAMDGPUCtorDtorLoweringLegacyPass());
1384
1385 if (TM.getTargetTriple().isAMDGCN() &&
1386 isPassEnabled(Opt: CLOpts.enable_image_intrinsic_optimizer))
1387 addPass(P: createAMDGPUImageIntrinsicOptimizerPass(&TM));
1388
1389 if (CLOpts.enable_uniform_intrinsic_combine)
1390 addPass(P: createAMDGPUUniformIntrinsicCombineLegacyPass());
1391
1392 // This can be disabled by passing ::Disable here or on the command line
1393 // with --expand-variadics-override=disable.
1394 addPass(P: createExpandVariadicsPass(ExpandVariadicsMode::Lowering));
1395
1396 // Function calls are not supported, so make sure we inline everything.
1397 addPass(P: createAMDGPUAlwaysInlinePass());
1398 addPass(P: createAlwaysInlinerLegacyPass());
1399
1400 // Handle uses of OpenCL image2d_t, image3d_t and sampler_t arguments.
1401 if (TM.getTargetTriple().getArch() == Triple::r600)
1402 addPass(P: createR600OpenCLImageTypeLoweringPass());
1403
1404 // Make enqueued block runtime handles externally visible.
1405 addPass(P: createAMDGPUExportKernelRuntimeHandlesLegacyPass());
1406
1407 // Lower special LDS accesses.
1408 if (CLOpts.enable_lower_exec_sync)
1409 addPass(P: createAMDGPULowerExecSyncLegacyPass());
1410
1411 // Lower LDS accesses to global memory pass if address sanitizer is enabled.
1412 if (CLOpts.enable_sw_lower_lds)
1413 addPass(P: createAMDGPUSwLowerLDSLegacyPass());
1414
1415 // Runs before PromoteAlloca so the latter can account for function uses
1416 if (EnableLowerModuleLDS) {
1417 addPass(P: createAMDGPULowerModuleLDSLegacyPass(TM: &TM));
1418 }
1419
1420 // Run atomic optimizer before Atomic Expand
1421 if ((TM.getTargetTriple().isAMDGCN()) &&
1422 (TM.getOptLevel() >= CodeGenOptLevel::Less) &&
1423 (CLOpts.atomic_optimizer_strategy != AMDGPU::ScanOptions::None)) {
1424 addPass(P: createAMDGPUAtomicOptimizerPass(ScanStrategy: CLOpts.atomic_optimizer_strategy));
1425 }
1426
1427 addPass(P: createAtomicExpandLegacyPass());
1428
1429 if (TM.getOptLevel() > CodeGenOptLevel::None) {
1430 addPass(P: createAMDGPUPromoteAlloca());
1431
1432 if (isPassEnabled(Opt: CLOpts.scalar_ir_passes))
1433 addStraightLineScalarOptimizationPasses();
1434
1435 if (CLOpts.enable_aa) {
1436 addPass(P: createAMDGPUAAWrapperPass());
1437 addPass(P: createExternalAAWrapperPass(Callback: [](Pass &P, Function &,
1438 AAResults &AAR) {
1439 if (auto *WrapperPass = P.getAnalysisIfAvailable<AMDGPUAAWrapperPass>())
1440 AAR.addAAResult(AAResult&: WrapperPass->getResult());
1441 }));
1442 }
1443
1444 if (TM.getTargetTriple().isAMDGCN()) {
1445 // TODO: May want to move later or split into an early and late one.
1446 addPass(
1447 P: createAMDGPUCodeGenPreparePass(ExpandDiv64InIR: CLOpts.codegenprepare_expand_div64));
1448 }
1449
1450 // Try to hoist loop invariant parts of divisions AMDGPUCodeGenPrepare may
1451 // have expanded.
1452 if (TM.getOptLevel() > CodeGenOptLevel::Less)
1453 addPass(P: createLICMPass());
1454 }
1455
1456 TargetPassConfig::addIRPasses();
1457
1458 // EarlyCSE is not always strong enough to clean up what LSR produces. For
1459 // example, GVN can combine
1460 //
1461 // %0 = add %a, %b
1462 // %1 = add %b, %a
1463 //
1464 // and
1465 //
1466 // %0 = shl nsw %a, 2
1467 // %1 = shl %a, 2
1468 //
1469 // but EarlyCSE can do neither of them.
1470 if (isPassEnabled(Opt: CLOpts.scalar_ir_passes))
1471 addEarlyCSEOrGVNPass();
1472}
1473
1474void AMDGPUPassConfig::addCodeGenPrepare() {
1475 if (TM->getTargetTriple().isAMDGCN() &&
1476 TM->getOptLevel() > CodeGenOptLevel::None)
1477 addPass(P: createAMDGPUPreloadKernelArgumentsLegacyPass(TM));
1478
1479 if (TM->getTargetTriple().isAMDGCN() && CLOpts.ir_lower_kernel_arguments)
1480 addPass(P: createAMDGPULowerKernelArgumentsPass());
1481
1482 TargetPassConfig::addCodeGenPrepare();
1483
1484 if (isPassEnabled(Opt: CLOpts.load_store_vectorizer))
1485 addPass(P: createLoadStoreVectorizerPass());
1486
1487 if (TM->getTargetTriple().isAMDGCN()) {
1488 // This lowering has been placed after codegenprepare to take advantage of
1489 // address mode matching (which is why it isn't put with the LDS lowerings).
1490 // It could be placed anywhere before uniformity annotations (an analysis
1491 // that it changes by splitting up fat pointers into their components)
1492 // but has been put before switch lowering and CFG flattening so that those
1493 // passes can run on the more optimized control flow this pass creates in
1494 // many cases.
1495 addPass(P: createAMDGPULowerBufferFatPointersPass());
1496 addPass(P: createAMDGPULowerIntrinsicsLegacyPass());
1497 }
1498
1499 // LowerSwitch pass may introduce unreachable blocks that can
1500 // cause unexpected behavior for subsequent passes. Placing it
1501 // here seems better that these blocks would get cleaned up by
1502 // UnreachableBlockElim inserted next in the pass flow.
1503 addPass(P: createLowerSwitchPass());
1504}
1505
1506bool AMDGPUPassConfig::addPreISel() {
1507 if (TM->getOptLevel() > CodeGenOptLevel::None)
1508 addPass(P: createFlattenCFGPass());
1509 return false;
1510}
1511
1512bool AMDGPUPassConfig::addInstSelector() {
1513 addPass(P: createAMDGPUISelDag(TM&: getAMDGPUTargetMachine(), OptLevel: getOptLevel()));
1514 return false;
1515}
1516
1517bool AMDGPUPassConfig::addGCPasses() {
1518 // Do nothing. GC is not supported.
1519 return false;
1520}
1521
1522//===----------------------------------------------------------------------===//
1523// GCN Legacy Pass Setup
1524//===----------------------------------------------------------------------===//
1525
1526bool GCNPassConfig::addPreISel() {
1527 AMDGPUPassConfig::addPreISel();
1528
1529 if (TM->getOptLevel() > CodeGenOptLevel::None) {
1530 addPass(P: createSinkingPass());
1531 addPass(P: createAMDGPULateCodeGenPrepareLegacyPass());
1532 }
1533
1534 // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
1535 // regions formed by them.
1536 addPass(PassID: &AMDGPUUnifyDivergentExitNodesID);
1537 addPass(P: createFixIrreduciblePass());
1538 addPass(P: createUnifyLoopExitsPass());
1539 addPass(P: createStructurizeCFGPass(/*SkipUniformRegions=*/true));
1540
1541 addPass(P: createAMDGPUAnnotateUniformValuesLegacy());
1542 addPass(P: createSIAnnotateControlFlowLegacyPass());
1543 // TODO: Move this right after structurizeCFG to avoid extra divergence
1544 // analysis. This depends on stopping SIAnnotateControlFlow from making
1545 // control flow modifications.
1546 addPass(P: createAMDGPURewriteUndefForPHILegacyPass());
1547
1548 // SDAG requires LCSSA, GlobalISel does not. Disable LCSSA for -global-isel
1549 // without any of the fallback options.
1550 if (getCGPassBuilderOption(TM: *TM).EnableGlobalISelOption !=
1551 cl::boolOrDefault::BOU_TRUE ||
1552 !isGlobalISelAbortEnabled())
1553 addPass(P: createLCSSAPass());
1554
1555 if (TM->getOptLevel() > CodeGenOptLevel::Less)
1556 addPass(PassID: &AMDGPUPerfHintAnalysisLegacyID);
1557
1558 return false;
1559}
1560
1561void GCNPassConfig::addMachineSSAOptimization() {
1562 TargetPassConfig::addMachineSSAOptimization();
1563
1564 // We want to fold operands after PeepholeOptimizer has run (or as part of
1565 // it), because it will eliminate extra copies making it easier to fold the
1566 // real source operand. We want to eliminate dead instructions after, so that
1567 // we see fewer uses of the copies. We then need to clean up the dead
1568 // instructions leftover after the operands are folded as well.
1569 //
1570 // XXX - Can we get away without running DeadMachineInstructionElim again?
1571 addPass(PassID: &SIFoldOperandsLegacyID);
1572 if (CLOpts.dpp_combine)
1573 addPass(PassID: &GCNDPPCombineLegacyID);
1574 addPass(PassID: &SILoadStoreOptimizerLegacyID);
1575 if (isPassEnabled(Opt: CLOpts.sdwa_peephole)) {
1576 addPass(PassID: &SIPeepholeSDWALegacyID);
1577 addPass(PassID: &EarlyMachineLICMID);
1578 addPass(PassID: &MachineCSELegacyID);
1579 addPass(PassID: &SIFoldOperandsLegacyID);
1580 }
1581 addPass(PassID: &DeadMachineInstructionElimID);
1582 addPass(P: createSIShrinkInstructionsLegacyPass());
1583}
1584
1585bool GCNPassConfig::addILPOpts() {
1586 if (CLOpts.early_ifcvt)
1587 addPass(PassID: &EarlyIfConverterLegacyID);
1588
1589 TargetPassConfig::addILPOpts();
1590 return false;
1591}
1592
1593bool GCNPassConfig::addInstSelector() {
1594 AMDGPUPassConfig::addInstSelector();
1595 addPass(PassID: &SIFixSGPRCopiesLegacyID);
1596 addPass(P: createSILowerI1CopiesLegacyPass());
1597 return false;
1598}
1599
1600bool GCNPassConfig::addIRTranslator() {
1601 addPass(P: new IRTranslatorLegacy(getOptLevel()));
1602 return false;
1603}
1604
1605void GCNPassConfig::addPreLegalizeMachineIR() {
1606 bool IsOptLevelNone = getOptLevel() == CodeGenOptLevel::None;
1607 addPass(P: createAMDGPUPreLegalizeCombinerLegacyPass(IsOptLevelNone));
1608 addPass(P: new LocalizerLegacy());
1609}
1610
1611bool GCNPassConfig::addLegalizeMachineIR() {
1612 addPass(P: new LegalizerLegacy());
1613 return false;
1614}
1615
1616void GCNPassConfig::addPreRegBankSelect() {
1617 bool IsOptNone = getOptLevel() == CodeGenOptLevel::None;
1618 addPass(P: createAMDGPUPostLegalizeCombinerLegacy(IsOptNone));
1619 addPass(P: createAMDGPUGlobalISelDivergenceLoweringPass());
1620}
1621
1622bool GCNPassConfig::addRegBankSelect() {
1623 addPass(P: createAMDGPURegBankSelectLegacyPass());
1624 addPass(P: createAMDGPURegBankLegalizeLegacyPass());
1625 return false;
1626}
1627
1628void GCNPassConfig::addPreGlobalInstructionSelect() {
1629 bool IsOptLevelNone = getOptLevel() == CodeGenOptLevel::None;
1630 addPass(P: createAMDGPURegBankCombinerLegacy(IsOptLevelNone));
1631}
1632
1633bool GCNPassConfig::addGlobalInstructionSelect() {
1634 addPass(P: new InstructionSelectLegacy(getOptLevel()));
1635 return false;
1636}
1637
1638void GCNPassConfig::addFastRegAlloc() {
1639 // FIXME: We have to disable the verifier here because of PHIElimination +
1640 // TwoAddressInstructions disabling it.
1641
1642 // This must be run immediately after phi elimination and before
1643 // TwoAddressInstructions, otherwise the processing of the tied operand of
1644 // SI_ELSE will introduce a copy of the tied operand source after the else.
1645 insertPass(TargetPassID: &PHIEliminationID, InsertedPassID: &SILowerControlFlowLegacyID);
1646
1647 insertPass(TargetPassID: &TwoAddressInstructionPassID, InsertedPassID: &SIWholeQuadModeID);
1648
1649 TargetPassConfig::addFastRegAlloc();
1650}
1651
1652void GCNPassConfig::addPreRegAlloc() {
1653 if (getOptLevel() != CodeGenOptLevel::None)
1654 addPass(PassID: &AMDGPUPrepareAGPRAllocLegacyID);
1655}
1656
1657void GCNPassConfig::addOptimizedRegAlloc() {
1658 if (CLOpts.dce_in_ra)
1659 insertPass(TargetPassID: &DetectDeadLanesID, InsertedPassID: &DeadMachineInstructionElimID);
1660
1661 if (CLOpts.opt_vgpr_liverange)
1662 insertPass(TargetPassID: &MachineLoopInfoID, InsertedPassID: &SIOptimizeVGPRLiveRangeLegacyID);
1663
1664 // This must be run immediately after phi elimination and before
1665 // TwoAddressInstructions, otherwise the processing of the tied operand of
1666 // SI_ELSE will introduce a copy of the tied operand source after the else.
1667 insertPass(TargetPassID: &PHIEliminationID, InsertedPassID: &SILowerControlFlowLegacyID);
1668
1669 if (CLOpts.enable_rewrite_partial_reg_uses)
1670 insertPass(TargetPassID: &RenameIndependentSubregsID, InsertedPassID: &GCNRewritePartialRegUsesID);
1671
1672 // Insertion point for passes depends on whether MachineScheduler is enabled.
1673 AnalysisID EndOfPreRA = CLOpts.use_ssa_machine_scheduler
1674 ? &RenameIndependentSubregsID
1675 : &MachineSchedulerID;
1676
1677 if (isPassEnabled(Opt: CLOpts.enable_pre_ra_optimizations))
1678 insertPass(TargetPassID: EndOfPreRA, InsertedPassID: &GCNPreRAOptimizationsID);
1679
1680 // Allow the scheduler to run before SIWholeQuadMode inserts exec manipulation
1681 // instructions that cause scheduling barriers.
1682 insertPass(TargetPassID: EndOfPreRA, InsertedPassID: &SIWholeQuadModeID);
1683
1684 if (CLOpts.opt_exec_mask_pre_ra)
1685 insertPass(TargetPassID: EndOfPreRA, InsertedPassID: &SIOptimizeExecMaskingPreRAID);
1686
1687 // This is not an essential optimization and it has a noticeable impact on
1688 // compilation time, so we only enable it from O2.
1689 if (TM->getOptLevel() > CodeGenOptLevel::Less)
1690 insertPass(TargetPassID: EndOfPreRA, InsertedPassID: &SIFormMemoryClausesID);
1691
1692 TargetPassConfig::addOptimizedRegAlloc();
1693}
1694
1695bool GCNPassConfig::addPreRewrite() {
1696 if (CLOpts.reassign_regs)
1697 addPass(PassID: &GCNNSAReassignID);
1698
1699 addPass(PassID: &AMDGPURewriteAGPRCopyMFMALegacyID);
1700 return true;
1701}
1702
1703FunctionPass *GCNPassConfig::createSGPRAllocPass(bool Optimized) {
1704 // Initialize the global default.
1705 llvm::call_once(flag&: InitializeDefaultSGPRRegisterAllocatorFlag,
1706 F&: initializeDefaultSGPRRegisterAllocatorOnce);
1707
1708 RegisterRegAlloc::FunctionPassCtor Ctor = SGPRRegisterRegAlloc::getDefault();
1709 if (Ctor != useDefaultRegisterAllocator)
1710 return Ctor();
1711
1712 if (Optimized)
1713 return createGreedyRegisterAllocator(F: onlyAllocateSGPRs);
1714
1715 return createFastRegisterAllocator(F: onlyAllocateSGPRs, ClearVirtRegs: false);
1716}
1717
1718FunctionPass *GCNPassConfig::createVGPRAllocPass(bool Optimized) {
1719 // Initialize the global default.
1720 llvm::call_once(flag&: InitializeDefaultVGPRRegisterAllocatorFlag,
1721 F&: initializeDefaultVGPRRegisterAllocatorOnce);
1722
1723 RegisterRegAlloc::FunctionPassCtor Ctor = VGPRRegisterRegAlloc::getDefault();
1724 if (Ctor != useDefaultRegisterAllocator)
1725 return Ctor();
1726
1727 if (Optimized)
1728 return createGreedyVGPRRegisterAllocator();
1729
1730 return createFastVGPRRegisterAllocator();
1731}
1732
1733FunctionPass *GCNPassConfig::createWWMRegAllocPass(bool Optimized) {
1734 // Initialize the global default.
1735 llvm::call_once(flag&: InitializeDefaultWWMRegisterAllocatorFlag,
1736 F&: initializeDefaultWWMRegisterAllocatorOnce);
1737
1738 RegisterRegAlloc::FunctionPassCtor Ctor = WWMRegisterRegAlloc::getDefault();
1739 if (Ctor != useDefaultRegisterAllocator)
1740 return Ctor();
1741
1742 if (Optimized)
1743 return createGreedyWWMRegisterAllocator();
1744
1745 return createFastWWMRegisterAllocator();
1746}
1747
1748FunctionPass *GCNPassConfig::createRegAllocPass(bool Optimized) {
1749 llvm_unreachable("should not be used");
1750}
1751
1752static const char RegAllocOptNotSupportedMessage[] =
1753 "-regalloc not supported with amdgcn. Use -sgpr-regalloc, -wwm-regalloc, "
1754 "and -vgpr-regalloc";
1755
1756bool GCNPassConfig::addRegAssignAndRewriteFast() {
1757 if (!usingDefaultRegAlloc())
1758 reportFatalUsageError(reason: RegAllocOptNotSupportedMessage);
1759
1760 addPass(PassID: &GCNPreRALongBranchRegID);
1761
1762 addPass(P: createSGPRAllocPass(Optimized: false));
1763
1764 // Equivalent of PEI for SGPRs.
1765 addPass(PassID: &SILowerSGPRSpillsLegacyID);
1766
1767 // To Allocate wwm registers used in whole quad mode operations (for shaders).
1768 addPass(PassID: &SIPreAllocateWWMRegsLegacyID);
1769
1770 // For allocating other wwm register operands.
1771 addPass(P: createWWMRegAllocPass(Optimized: false));
1772
1773 addPass(PassID: &SILowerWWMCopiesLegacyID);
1774 addPass(PassID: &AMDGPUReserveWWMRegsLegacyID);
1775
1776 // For allocating per-thread VGPRs.
1777 addPass(P: createVGPRAllocPass(Optimized: false));
1778
1779 return true;
1780}
1781
1782bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
1783 if (!usingDefaultRegAlloc())
1784 reportFatalUsageError(reason: RegAllocOptNotSupportedMessage);
1785
1786 addPass(PassID: &GCNPreRALongBranchRegID);
1787
1788 addPass(P: createSGPRAllocPass(Optimized: true));
1789
1790 // Commit allocated register changes. This is mostly necessary because too
1791 // many things rely on the use lists of the physical registers, such as the
1792 // verifier. This is only necessary with allocators which use LiveIntervals,
1793 // since FastRegAlloc does the replacements itself.
1794 addPass(P: createVirtRegRewriter(ClearVirtRegs: false));
1795
1796 // At this point, the sgpr-regalloc has been done and it is good to have the
1797 // stack slot coloring to try to optimize the SGPR spill stack indices before
1798 // attempting the custom SGPR spill lowering.
1799 addPass(PassID: &StackSlotColoringID);
1800
1801 // Equivalent of PEI for SGPRs.
1802 addPass(PassID: &SILowerSGPRSpillsLegacyID);
1803
1804 // To Allocate wwm registers used in whole quad mode operations (for shaders).
1805 addPass(PassID: &SIPreAllocateWWMRegsLegacyID);
1806
1807 // For allocating other whole wave mode registers.
1808 addPass(P: createWWMRegAllocPass(Optimized: true));
1809 addPass(PassID: &SILowerWWMCopiesLegacyID);
1810 addPass(P: createVirtRegRewriter(ClearVirtRegs: false));
1811 addPass(PassID: &AMDGPUReserveWWMRegsLegacyID);
1812
1813 // For allocating per-thread VGPRs.
1814 addPass(P: createVGPRAllocPass(Optimized: true));
1815
1816 addPreRewrite();
1817 addPass(PassID: &VirtRegRewriterID);
1818
1819 addPass(P: createRegAllocScoringPass());
1820
1821 addPass(PassID: &AMDGPUMarkLastScratchLoadID);
1822
1823 return true;
1824}
1825
1826void GCNPassConfig::addPostRegAlloc() {
1827 addPass(PassID: &SIFixVGPRCopiesID);
1828 if (getOptLevel() > CodeGenOptLevel::None)
1829 addPass(PassID: &SIOptimizeExecMaskingLegacyID);
1830 TargetPassConfig::addPostRegAlloc();
1831}
1832
1833void GCNPassConfig::addPreSched2() {
1834 if (TM->getOptLevel() > CodeGenOptLevel::None)
1835 addPass(P: createSIShrinkInstructionsLegacyPass());
1836 addPass(PassID: &SIPostRABundlerLegacyID);
1837}
1838
1839static bool mayNeedAsyncBarrierArriveWorkaround(const Triple &TT) {
1840 Triple::SubArchType SubArch = TT.getSubArch();
1841 return SubArch == Triple::NoSubArch ||
1842 SubArch == Triple::AMDGPUSubArch1250S ||
1843 AMDGPU::getMajorSubArch(SubArch) == Triple::AMDGPUSubArch12_5;
1844}
1845
1846void GCNPassConfig::addPreEmitPass() {
1847 if (isPassEnabled(Opt: CLOpts.enable_vopd, Level: CodeGenOptLevel::Less))
1848 addPass(PassID: &GCNCreateVOPDID);
1849 addPass(P: createSIMemoryLegalizerPass());
1850 if (getOptLevel() > CodeGenOptLevel::None)
1851 addPass(PassID: &SIPostRA16BitMovFoldingLegacyID);
1852 if (mayNeedAsyncBarrierArriveWorkaround(TT: TM->getTargetTriple()))
1853 addPass(P: createSIInsertAsyncBarrierArriveWorkaroundPass());
1854 addPass(P: createSIInsertWaitcntsPass());
1855
1856 addPass(P: createSIModeRegisterPass());
1857
1858 if (getOptLevel() > CodeGenOptLevel::None)
1859 addPass(PassID: &SIInsertHardClausesID);
1860
1861 addPass(PassID: &SILateBranchLoweringPassID);
1862 if (CLOpts.set_wave_priority)
1863 addPass(P: createAMDGPUSetWavePriorityPass());
1864 if (getOptLevel() > CodeGenOptLevel::None)
1865 addPass(PassID: &SIPreEmitPeepholeID);
1866 // The hazard recognizer that runs as part of the post-ra scheduler does not
1867 // guarantee to be able handle all hazards correctly. This is because if there
1868 // are multiple scheduling regions in a basic block, the regions are scheduled
1869 // bottom up, so when we begin to schedule a region we don't know what
1870 // instructions were emitted directly before it.
1871 //
1872 // Here we add a stand-alone hazard recognizer pass which can handle all
1873 // cases.
1874 addPass(PassID: &PostRAHazardRecognizerID);
1875
1876 addPass(PassID: &AMDGPUWaitSGPRHazardsLegacyID);
1877
1878 addPass(PassID: &AMDGPULowerVGPREncodingLegacyID);
1879
1880 if (isPassEnabled(Opt: CLOpts.enable_delay_alu, Level: CodeGenOptLevel::Less))
1881 addPass(PassID: &AMDGPUInsertDelayAluID);
1882
1883 addPass(PassID: &BranchRelaxationPassID);
1884}
1885
1886void GCNPassConfig::addPostBBSections() {
1887 // We run this later to avoid passes like livedebugvalues and BBSections
1888 // having to deal with the apparent multi-entry functions we may generate.
1889 addPass(P: createAMDGPUPreloadKernArgPrologLegacyPass());
1890}
1891
1892TargetPassConfig *GCNTargetMachine::createPassConfig(PassManagerBase &PM) {
1893 return new GCNPassConfig(*this, PM);
1894}
1895
1896void GCNTargetMachine::registerMachineRegisterInfoCallback(
1897 MachineFunction &MF) const {
1898 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1899 MF.getRegInfo().addDelegate(delegate: MFI);
1900}
1901
1902MachineFunctionInfo *GCNTargetMachine::createMachineFunctionInfo(
1903 BumpPtrAllocator &Allocator, const Function &F,
1904 const TargetSubtargetInfo *STI) const {
1905 return SIMachineFunctionInfo::create<SIMachineFunctionInfo>(
1906 Allocator, F, STI: static_cast<const GCNSubtarget *>(STI));
1907}
1908
1909yaml::MachineFunctionInfo *GCNTargetMachine::createDefaultFuncInfoYAML() const {
1910 return new yaml::SIMachineFunctionInfo();
1911}
1912
1913yaml::MachineFunctionInfo *
1914GCNTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
1915 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1916 return new yaml::SIMachineFunctionInfo(
1917 *MFI, *MF.getSubtarget<GCNSubtarget>().getRegisterInfo(), MF);
1918}
1919
1920bool GCNTargetMachine::parseMachineFunctionInfo(
1921 const yaml::MachineFunctionInfo &MFI_, PerFunctionMIParsingState &PFS,
1922 SMDiagnostic &Error, SMRange &SourceRange) const {
1923 const yaml::SIMachineFunctionInfo &YamlMFI =
1924 static_cast<const yaml::SIMachineFunctionInfo &>(MFI_);
1925 MachineFunction &MF = PFS.MF;
1926 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1927 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1928
1929 if (MFI->initializeBaseYamlFields(YamlMFI, MF, PFS, Error, SourceRange))
1930 return true;
1931
1932 if (MFI->Occupancy == 0) {
1933 // Fixup the subtarget dependent default value.
1934 MFI->Occupancy = ST.getOccupancyWithWorkGroupSizes(MF).second;
1935 }
1936
1937 auto parseRegister = [&](const yaml::StringValue &RegName, Register &RegVal) {
1938 Register TempReg;
1939 if (parseNamedRegisterReference(PFS, Reg&: TempReg, Src: RegName.Value, Error)) {
1940 SourceRange = RegName.SourceRange;
1941 return true;
1942 }
1943 RegVal = TempReg;
1944
1945 return false;
1946 };
1947
1948 auto parseOptionalRegister = [&](const yaml::StringValue &RegName,
1949 Register &RegVal) {
1950 return !RegName.Value.empty() && parseRegister(RegName, RegVal);
1951 };
1952
1953 if (parseOptionalRegister(YamlMFI.VGPRForAGPRCopy, MFI->VGPRForAGPRCopy))
1954 return true;
1955
1956 if (parseOptionalRegister(YamlMFI.SGPRForEXECCopy, MFI->SGPRForEXECCopy))
1957 return true;
1958
1959 if (parseOptionalRegister(YamlMFI.LongBranchReservedReg,
1960 MFI->LongBranchReservedReg))
1961 return true;
1962
1963 auto diagnoseRegisterClass = [&](const yaml::StringValue &RegName) {
1964 // Create a diagnostic for a the register string literal.
1965 const MemoryBuffer &Buffer =
1966 *PFS.SM->getMemoryBuffer(i: PFS.SM->getMainFileID());
1967 Error = SMDiagnostic(
1968 *PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1,
1969 static_cast<int>(RegName.Value.size()), SourceMgr::DK_Error,
1970 "incorrect register class for field", RegName.Value, {}, {});
1971 SourceRange = RegName.SourceRange;
1972 return true;
1973 };
1974
1975 if (parseRegister(YamlMFI.ScratchRSrcReg, MFI->ScratchRSrcReg) ||
1976 parseRegister(YamlMFI.FrameOffsetReg, MFI->FrameOffsetReg) ||
1977 parseRegister(YamlMFI.StackPtrOffsetReg, MFI->StackPtrOffsetReg))
1978 return true;
1979
1980 if (MFI->ScratchRSrcReg != AMDGPU::PRIVATE_RSRC_REG &&
1981 !AMDGPU::SGPR_128RegClass.contains(Reg: MFI->ScratchRSrcReg)) {
1982 return diagnoseRegisterClass(YamlMFI.ScratchRSrcReg);
1983 }
1984
1985 if (MFI->FrameOffsetReg != AMDGPU::FP_REG &&
1986 !AMDGPU::SGPR_32RegClass.contains(Reg: MFI->FrameOffsetReg)) {
1987 return diagnoseRegisterClass(YamlMFI.FrameOffsetReg);
1988 }
1989
1990 if (MFI->StackPtrOffsetReg != AMDGPU::SP_REG &&
1991 !AMDGPU::SGPR_32RegClass.contains(Reg: MFI->StackPtrOffsetReg)) {
1992 return diagnoseRegisterClass(YamlMFI.StackPtrOffsetReg);
1993 }
1994
1995 for (const auto &YamlReg : YamlMFI.WWMReservedRegs) {
1996 Register ParsedReg;
1997 if (parseRegister(YamlReg, ParsedReg))
1998 return true;
1999
2000 MFI->reserveWWMRegister(Reg: ParsedReg);
2001 }
2002
2003 for (const auto &[_, Info] : PFS.VRegInfosNamed) {
2004 MFI->setFlag(Reg: Info->VReg, Flag: Info->Flags);
2005 }
2006 for (const auto &[_, Info] : PFS.VRegInfos) {
2007 MFI->setFlag(Reg: Info->VReg, Flag: Info->Flags);
2008 }
2009
2010 for (const auto &YamlRegStr : YamlMFI.SpillPhysVGPRS) {
2011 Register ParsedReg;
2012 if (parseRegister(YamlRegStr, ParsedReg))
2013 return true;
2014 MFI->SpillPhysVGPRs.push_back(Elt: ParsedReg);
2015 }
2016
2017 auto parseAndCheckArgument = [&](const std::optional<yaml::SIArgument> &A,
2018 const TargetRegisterClass &RC,
2019 ArgDescriptor &Arg, unsigned UserSGPRs,
2020 unsigned SystemSGPRs) {
2021 // Skip parsing if it's not present.
2022 if (!A)
2023 return false;
2024
2025 if (A->IsRegister) {
2026 Register Reg;
2027 if (parseNamedRegisterReference(PFS, Reg, Src: A->RegisterName.Value, Error)) {
2028 SourceRange = A->RegisterName.SourceRange;
2029 return true;
2030 }
2031 if (!RC.contains(Reg))
2032 return diagnoseRegisterClass(A->RegisterName);
2033 Arg = ArgDescriptor::createRegister(Reg);
2034 } else
2035 Arg = ArgDescriptor::createStack(Offset: A->StackOffset);
2036 // Check and apply the optional mask.
2037 if (A->Mask)
2038 Arg = ArgDescriptor::createArg(Arg, Mask: *A->Mask);
2039
2040 MFI->NumUserSGPRs += UserSGPRs;
2041 MFI->NumSystemSGPRs += SystemSGPRs;
2042 return false;
2043 };
2044
2045 if (YamlMFI.ArgInfo &&
2046 (parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentBuffer,
2047 AMDGPU::SGPR_128RegClass,
2048 MFI->ArgInfo.PrivateSegmentBuffer, 4, 0) ||
2049 parseAndCheckArgument(YamlMFI.ArgInfo->DispatchPtr,
2050 AMDGPU::SReg_64RegClass, MFI->ArgInfo.DispatchPtr,
2051 2, 0) ||
2052 parseAndCheckArgument(YamlMFI.ArgInfo->QueuePtr, AMDGPU::SReg_64RegClass,
2053 MFI->ArgInfo.QueuePtr, 2, 0) ||
2054 parseAndCheckArgument(YamlMFI.ArgInfo->KernargSegmentPtr,
2055 AMDGPU::SReg_64RegClass,
2056 MFI->ArgInfo.KernargSegmentPtr, 2, 0) ||
2057 parseAndCheckArgument(YamlMFI.ArgInfo->DispatchID,
2058 AMDGPU::SReg_64RegClass, MFI->ArgInfo.DispatchID,
2059 2, 0) ||
2060 parseAndCheckArgument(YamlMFI.ArgInfo->FlatScratchInit,
2061 AMDGPU::SReg_64RegClass,
2062 MFI->ArgInfo.FlatScratchInit, 2, 0) ||
2063 parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentSize,
2064 AMDGPU::SGPR_32RegClass,
2065 MFI->ArgInfo.PrivateSegmentSize, 1, 0) ||
2066 parseAndCheckArgument(YamlMFI.ArgInfo->LDSKernelId,
2067 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.LDSKernelId,
2068 1, 0) ||
2069 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDX,
2070 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDX,
2071 0, 1) ||
2072 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDY,
2073 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDY,
2074 0, 1) ||
2075 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDZ,
2076 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDZ,
2077 0, 1) ||
2078 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupInfo,
2079 AMDGPU::SGPR_32RegClass,
2080 MFI->ArgInfo.WorkGroupInfo, 0, 1) ||
2081 parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentWaveByteOffset,
2082 AMDGPU::SGPR_32RegClass,
2083 MFI->ArgInfo.PrivateSegmentWaveByteOffset, 0, 1) ||
2084 parseAndCheckArgument(YamlMFI.ArgInfo->ImplicitArgPtr,
2085 AMDGPU::SReg_64RegClass,
2086 MFI->ArgInfo.ImplicitArgPtr, 0, 0) ||
2087 parseAndCheckArgument(YamlMFI.ArgInfo->ImplicitBufferPtr,
2088 AMDGPU::SReg_64RegClass,
2089 MFI->ArgInfo.ImplicitBufferPtr, 2, 0) ||
2090 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDX,
2091 AMDGPU::VGPR_32RegClass, MFI->ArgInfo.WorkItemIDX,
2092 0, 0) ||
2093 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDY,
2094 AMDGPU::VGPR_32RegClass, MFI->ArgInfo.WorkItemIDY,
2095 0, 0) ||
2096 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDZ,
2097 AMDGPU::VGPR_32RegClass, MFI->ArgInfo.WorkItemIDZ,
2098 0, 0)))
2099 return true;
2100
2101 // Parse FirstKernArgPreloadReg separately, since it's a Register,
2102 // not ArgDescriptor.
2103 if (YamlMFI.ArgInfo && YamlMFI.ArgInfo->FirstKernArgPreloadReg) {
2104 const yaml::SIArgument &A = *YamlMFI.ArgInfo->FirstKernArgPreloadReg;
2105
2106 if (!A.IsRegister) {
2107 // For stack arguments, we don't have RegisterName.SourceRange,
2108 // but we should have some location info from the YAML parser
2109 const MemoryBuffer &Buffer =
2110 *PFS.SM->getMemoryBuffer(i: PFS.SM->getMainFileID());
2111 // Create a minimal valid source range
2112 SMLoc Loc = SMLoc::getFromPointer(Ptr: Buffer.getBufferStart());
2113 SMRange Range(Loc, Loc);
2114
2115 Error = SMDiagnostic(
2116 *PFS.SM, Loc, Buffer.getBufferIdentifier(), 1, 0, SourceMgr::DK_Error,
2117 "firstKernArgPreloadReg must be a register, not a stack location", "",
2118 {}, {});
2119
2120 SourceRange = Range;
2121 return true;
2122 }
2123
2124 Register Reg;
2125 if (parseNamedRegisterReference(PFS, Reg, Src: A.RegisterName.Value, Error)) {
2126 SourceRange = A.RegisterName.SourceRange;
2127 return true;
2128 }
2129
2130 if (!AMDGPU::SGPR_32RegClass.contains(Reg))
2131 return diagnoseRegisterClass(A.RegisterName);
2132
2133 MFI->ArgInfo.FirstKernArgPreloadReg = Reg;
2134 MFI->NumUserSGPRs += YamlMFI.NumKernargPreloadSGPRs;
2135 }
2136
2137 if (ST.hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode)) {
2138 MFI->Mode.IEEE = YamlMFI.Mode.IEEE;
2139 MFI->Mode.DX10Clamp = YamlMFI.Mode.DX10Clamp;
2140 }
2141
2142 // FIXME: Move proper support for denormal-fp-math into base MachineFunction
2143 MFI->Mode.FP32Denormals.Input = YamlMFI.Mode.FP32InputDenormals
2144 ? DenormalMode::IEEE
2145 : DenormalMode::PreserveSign;
2146 MFI->Mode.FP32Denormals.Output = YamlMFI.Mode.FP32OutputDenormals
2147 ? DenormalMode::IEEE
2148 : DenormalMode::PreserveSign;
2149
2150 MFI->Mode.FP64FP16Denormals.Input = YamlMFI.Mode.FP64FP16InputDenormals
2151 ? DenormalMode::IEEE
2152 : DenormalMode::PreserveSign;
2153 MFI->Mode.FP64FP16Denormals.Output = YamlMFI.Mode.FP64FP16OutputDenormals
2154 ? DenormalMode::IEEE
2155 : DenormalMode::PreserveSign;
2156
2157 if (YamlMFI.HasInitWholeWave)
2158 MFI->setInitWholeWave();
2159
2160 return false;
2161}
2162
2163//===----------------------------------------------------------------------===//
2164// AMDGPU CodeGen Pass Builder interface.
2165//===----------------------------------------------------------------------===//
2166
2167AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
2168 GCNTargetMachine &TM, const CGPassBuilderOption &Opts,
2169 PassInstrumentationCallbacks *PIC)
2170 : CodeGenPassBuilder(TM, Opts, PIC), CLOpts(TM.getCLOpts()) {
2171 Opt.MISchedPostRA = true;
2172 Opt.RequiresCodeGenSCCOrder = true;
2173 Opt.EnableSSAMachineScheduler = CLOpts.use_ssa_machine_scheduler;
2174 // Exceptions and StackMaps are not supported, so these passes will never do
2175 // anything.
2176 // Garbage collection is not supported.
2177 disablePass<StackMapLivenessPass, FuncletLayoutPass, PatchableFunctionPass,
2178 ShadowStackGCLoweringPass, GCLoweringPass>();
2179
2180 // If enabled, use the SSA machine scheduler instead of the regular machine
2181 // scheduler.
2182 if (Opt.EnableSSAMachineScheduler)
2183 disablePass<MachineSchedulerPass>();
2184}
2185
2186void AMDGPUCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
2187 if (CLOpts.enable_remove_incompatible_functions &&
2188 TM.getTargetTriple().isAMDGCN()) {
2189 flushFPMsToMPM(PMW);
2190 addModulePass(Pass: AMDGPURemoveIncompatibleFunctionsPass(TM), PMW);
2191 }
2192
2193 flushFPMsToMPM(PMW);
2194
2195 if (TM.getTargetTriple().isAMDGCN())
2196 addModulePass(Pass: AMDGPUPrintfRuntimeBindingPass(), PMW);
2197
2198 if (CLOpts.lower_global_ctor_dtor)
2199 addModulePass(Pass: AMDGPUCtorDtorLoweringPass(), PMW);
2200
2201 if (isPassEnabled(Opt: CLOpts.enable_image_intrinsic_optimizer))
2202 addFunctionPass(Pass: AMDGPUImageIntrinsicOptimizerPass(TM), PMW);
2203
2204 if (CLOpts.enable_uniform_intrinsic_combine)
2205 addFunctionPass(Pass: AMDGPUUniformIntrinsicCombinePass(), PMW);
2206 // This can be disabled by passing ::Disable here or on the command line
2207 // with --expand-variadics-override=disable.
2208 flushFPMsToMPM(PMW);
2209 addModulePass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Lowering), PMW);
2210
2211 addModulePass(Pass: AMDGPUAlwaysInlinePass(), PMW);
2212 addModulePass(Pass: AlwaysInlinerPass(), PMW);
2213
2214 addModulePass(Pass: AMDGPUExportKernelRuntimeHandlesPass(), PMW);
2215
2216 if (CLOpts.enable_lower_exec_sync)
2217 addModulePass(Pass: AMDGPULowerExecSyncPass(), PMW);
2218
2219 if (CLOpts.enable_sw_lower_lds)
2220 addModulePass(Pass: AMDGPUSwLowerLDSPass(), PMW);
2221
2222 // Runs before PromoteAlloca so the latter can account for function uses
2223 if (EnableLowerModuleLDS)
2224 addModulePass(Pass: AMDGPULowerModuleLDSPass(getTM()), PMW);
2225
2226 // Run atomic optimizer before Atomic Expand
2227 if (TM.getOptLevel() >= CodeGenOptLevel::Less &&
2228 (CLOpts.atomic_optimizer_strategy != AMDGPU::ScanOptions::None))
2229 addFunctionPass(
2230 Pass: AMDGPUAtomicOptimizerPass(TM, CLOpts.atomic_optimizer_strategy), PMW);
2231
2232 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
2233
2234 if (TM.getOptLevel() > CodeGenOptLevel::None) {
2235 addFunctionPass(Pass: AMDGPUPromoteAllocaPass(TM), PMW);
2236 if (isPassEnabled(Opt: CLOpts.scalar_ir_passes))
2237 addStraightLineScalarOptimizationPasses(PMW);
2238
2239 // TODO: Handle CLOpts.enable_aa
2240
2241 // TODO: May want to move later or split into an early and late one.
2242 addFunctionPass(Pass: AMDGPUCodeGenPreparePass(TM), PMW);
2243
2244 // Try to hoist loop invariant parts of divisions AMDGPUCodeGenPrepare may
2245 // have expanded.
2246 if (TM.getOptLevel() > CodeGenOptLevel::Less) {
2247 addFunctionPass(Pass: createFunctionToLoopPassAdaptor(Pass: LICMPass(LICMOptions()),
2248 /*UseMemorySSA=*/true),
2249 PMW);
2250 }
2251 }
2252
2253 Base::addIRPasses(PMW);
2254
2255 // EarlyCSE is not always strong enough to clean up what LSR produces. For
2256 // example, GVN can combine
2257 //
2258 // %0 = add %a, %b
2259 // %1 = add %b, %a
2260 //
2261 // and
2262 //
2263 // %0 = shl nsw %a, 2
2264 // %1 = shl %a, 2
2265 //
2266 // but EarlyCSE can do neither of them.
2267 if (isPassEnabled(Opt: CLOpts.scalar_ir_passes))
2268 addEarlyCSEOrGVNPass(PMW);
2269}
2270
2271void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(PassManagerWrapper &PMW) {
2272 if (TM.getOptLevel() > CodeGenOptLevel::None) {
2273 flushFPMsToMPM(PMW);
2274 addModulePass(Pass: AMDGPUPreloadKernelArgumentsPass(TM), PMW);
2275 }
2276
2277 if (CLOpts.ir_lower_kernel_arguments)
2278 addFunctionPass(Pass: AMDGPULowerKernelArgumentsPass(TM), PMW);
2279
2280 Base::addCodeGenPrepare(PMW);
2281
2282 if (isPassEnabled(Opt: CLOpts.load_store_vectorizer))
2283 addFunctionPass(Pass: LoadStoreVectorizerPass(), PMW);
2284
2285 // This lowering has been placed after codegenprepare to take advantage of
2286 // address mode matching (which is why it isn't put with the LDS lowerings).
2287 // It could be placed anywhere before uniformity annotations (an analysis
2288 // that it changes by splitting up fat pointers into their components)
2289 // but has been put before switch lowering and CFG flattening so that those
2290 // passes can run on the more optimized control flow this pass creates in
2291 // many cases.
2292 flushFPMsToMPM(PMW);
2293 addModulePass(Pass: AMDGPULowerBufferFatPointersPass(TM), PMW);
2294 flushFPMsToMPM(PMW);
2295 requireCGSCCOrder(PMW);
2296
2297 addModulePass(Pass: AMDGPULowerIntrinsicsPass(getTM()), PMW);
2298
2299 // LowerSwitch pass may introduce unreachable blocks that can cause unexpected
2300 // behavior for subsequent passes. Placing it here seems better that these
2301 // blocks would get cleaned up by UnreachableBlockElim inserted next in the
2302 // pass flow.
2303 addFunctionPass(Pass: LowerSwitchPass(), PMW);
2304}
2305
2306void AMDGPUCodeGenPassBuilder::addPreISel(PassManagerWrapper &PMW) {
2307
2308 if (TM.getOptLevel() > CodeGenOptLevel::None) {
2309 addFunctionPass(Pass: FlattenCFGPass(), PMW);
2310 addFunctionPass(Pass: SinkingPass(), PMW);
2311 addFunctionPass(Pass: AMDGPULateCodeGenPreparePass(getTM()), PMW);
2312 }
2313
2314 // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
2315 // regions formed by them.
2316
2317 addFunctionPass(Pass: AMDGPUUnifyDivergentExitNodesPass(), PMW);
2318 addFunctionPass(Pass: FixIrreduciblePass(), PMW);
2319 addFunctionPass(Pass: UnifyLoopExitsPass(), PMW);
2320 addFunctionPass(Pass: StructurizeCFGPass(/*SkipUniformRegions=*/true), PMW);
2321
2322 addFunctionPass(Pass: AMDGPUAnnotateUniformValuesPass(), PMW);
2323
2324 addFunctionPass(Pass: SIAnnotateControlFlowPass(getTM()), PMW);
2325
2326 // TODO: Move this right after structurizeCFG to avoid extra divergence
2327 // analysis. This depends on stopping SIAnnotateControlFlow from making
2328 // control flow modifications.
2329 addFunctionPass(Pass: AMDGPURewriteUndefForPHIPass(), PMW);
2330
2331 if (getCGPassBuilderOption(TM).EnableGlobalISelOption !=
2332 cl::boolOrDefault::BOU_TRUE ||
2333 !isGlobalISelAbortEnabled())
2334 addFunctionPass(Pass: LCSSAPass(), PMW);
2335
2336 if (TM.getOptLevel() > CodeGenOptLevel::Less) {
2337 flushFPMsToMPM(PMW);
2338 addModulePass(Pass: AMDGPUPerfHintAnalysisPass(getTM()), PMW);
2339 }
2340}
2341
2342void AMDGPUCodeGenPassBuilder::addILPOpts(PassManagerWrapper &PMW) {
2343 if (CLOpts.early_ifcvt)
2344 addMachineFunctionPass(Pass: EarlyIfConverterPass(), PMW);
2345
2346 Base::addILPOpts(PMW);
2347}
2348
2349void AMDGPUCodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
2350 addModulePass(Pass: AMDGPUAsmPrinterBeginPass(), PMW,
2351 /*Force=*/true);
2352}
2353
2354void AMDGPUCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
2355 addMachineFunctionPass(Pass: AMDGPUAsmPrinterPass(), PMW);
2356}
2357
2358void AMDGPUCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
2359 addModulePass(Pass: AMDGPUAsmPrinterEndPass(), PMW);
2360}
2361
2362Error AMDGPUCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
2363 addMachineFunctionPass(Pass: AMDGPUISelDAGToDAGPass(TM), PMW);
2364 addMachineFunctionPass(Pass: SIFixSGPRCopiesPass(), PMW);
2365 addMachineFunctionPass(Pass: SILowerI1CopiesPass(), PMW);
2366 return Error::success();
2367}
2368
2369Error AMDGPUCodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
2370 addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW);
2371 return Error::success();
2372}
2373
2374void AMDGPUCodeGenPassBuilder::addPreLegalizeMachineIR(
2375 PassManagerWrapper &PMW) {
2376 addMachineFunctionPass(Pass: AMDGPUPreLegalizerCombinerPass(), PMW);
2377 addMachineFunctionPass(Pass: LocalizerPass(), PMW);
2378}
2379
2380Error AMDGPUCodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
2381 addMachineFunctionPass(Pass: LegalizerPass(), PMW);
2382 return Error::success();
2383}
2384
2385void AMDGPUCodeGenPassBuilder::addPreRegBankSelect(PassManagerWrapper &PMW) {
2386 addMachineFunctionPass(Pass: AMDGPUPostLegalizerCombinerPass(), PMW);
2387 addMachineFunctionPass(Pass: AMDGPUGlobalISelDivergenceLoweringPass(), PMW);
2388}
2389
2390Error AMDGPUCodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
2391 addMachineFunctionPass(Pass: AMDGPURegBankSelectPass(), PMW);
2392 addMachineFunctionPass(Pass: AMDGPURegBankLegalizePass(), PMW);
2393 return Error::success();
2394}
2395
2396void AMDGPUCodeGenPassBuilder::addPreGlobalInstructionSelect(
2397 PassManagerWrapper &PMW) {
2398 bool IsOptLevelNone = getOptLevel() == CodeGenOptLevel::None;
2399 addMachineFunctionPass(Pass: AMDGPURegBankCombinerPass(IsOptLevelNone), PMW);
2400}
2401
2402Error AMDGPUCodeGenPassBuilder::addGlobalInstructionSelect(
2403 PassManagerWrapper &PMW) {
2404 addMachineFunctionPass(Pass: InstructionSelectPass(getOptLevel()), PMW);
2405 return Error::success();
2406}
2407
2408void AMDGPUCodeGenPassBuilder::addPreRewrite(PassManagerWrapper &PMW) {
2409 if (CLOpts.reassign_regs) {
2410 addMachineFunctionPass(Pass: GCNNSAReassignPass(), PMW);
2411 }
2412
2413 addMachineFunctionPass(Pass: AMDGPURewriteAGPRCopyMFMAPass(), PMW);
2414}
2415
2416void AMDGPUCodeGenPassBuilder::addMachineSSAOptimization(
2417 PassManagerWrapper &PMW) {
2418 Base::addMachineSSAOptimization(PMW);
2419
2420 addMachineFunctionPass(Pass: SIFoldOperandsPass(), PMW);
2421 if (CLOpts.dpp_combine) {
2422 addMachineFunctionPass(Pass: GCNDPPCombinePass(), PMW);
2423 }
2424 addMachineFunctionPass(Pass: SILoadStoreOptimizerPass(), PMW);
2425 if (isPassEnabled(Opt: CLOpts.sdwa_peephole)) {
2426 addMachineFunctionPass(Pass: SIPeepholeSDWAPass(), PMW);
2427 addMachineFunctionPass(Pass: EarlyMachineLICMPass(), PMW);
2428 addMachineFunctionPass(Pass: MachineCSEPass(), PMW);
2429 addMachineFunctionPass(Pass: SIFoldOperandsPass(), PMW);
2430 }
2431 addMachineFunctionPass(Pass: DeadMachineInstructionElimPass(), PMW);
2432 addMachineFunctionPass(Pass: SIShrinkInstructionsPass(), PMW);
2433}
2434
2435Error AMDGPUCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
2436 insertPass<PHIEliminationPass>(Pass: SILowerControlFlowPass());
2437
2438 insertPass<TwoAddressInstructionPass>(Pass: SIWholeQuadModePass());
2439
2440 return Base::addFastRegAlloc(PMW);
2441}
2442
2443Error AMDGPUCodeGenPassBuilder::addRegAssignAndRewriteFast(
2444 PassManagerWrapper &PMW) {
2445 if (auto Err = validateRegAllocOptions())
2446 return Err;
2447
2448 addMachineFunctionPass(Pass: GCNPreRALongBranchRegPass(), PMW);
2449
2450 // SGPR allocation - default to fast at -O0.
2451 if (SGPRRegAllocNPM == RegAllocType::Greedy)
2452 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateSGPRs, "sgpr"}), PMW);
2453 else
2454 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateSGPRs, "sgpr", false}),
2455 PMW);
2456
2457 // Equivalent of PEI for SGPRs.
2458 addMachineFunctionPass(Pass: SILowerSGPRSpillsPass(), PMW);
2459
2460 // To Allocate wwm registers used in whole quad mode operations (for shaders).
2461 addMachineFunctionPass(Pass: SIPreAllocateWWMRegsPass(), PMW);
2462
2463 // WWM allocation - default to fast at -O0.
2464 if (WWMRegAllocNPM == RegAllocType::Greedy)
2465 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateWWMRegs, "wwm"}), PMW);
2466 else
2467 addMachineFunctionPass(
2468 Pass: RegAllocFastPass({onlyAllocateWWMRegs, "wwm", false}), PMW);
2469
2470 addMachineFunctionPass(Pass: SILowerWWMCopiesPass(), PMW);
2471 addMachineFunctionPass(Pass: AMDGPUReserveWWMRegsPass(), PMW);
2472
2473 // VGPR allocation - default to fast at -O0.
2474 if (VGPRRegAllocNPM == RegAllocType::Greedy)
2475 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2476 else
2477 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2478
2479 return Error::success();
2480}
2481
2482Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
2483 if (CLOpts.dce_in_ra)
2484 insertPass<DetectDeadLanesPass>(Pass: DeadMachineInstructionElimPass());
2485
2486 if (CLOpts.opt_vgpr_liverange)
2487 insertPass<RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>>(
2488 Pass: SIOptimizeVGPRLiveRangePass());
2489
2490 // This must be run immediately after phi elimination and before
2491 // TwoAddressInstructions, otherwise the processing of the tied operand of
2492 // SI_ELSE will introduce a copy of the tied operand source after the else.
2493 insertPass<PHIEliminationPass>(Pass: SILowerControlFlowPass());
2494
2495 if (CLOpts.enable_rewrite_partial_reg_uses)
2496 insertPass<RenameIndependentSubregsPass>(Pass: GCNRewritePartialRegUsesPass());
2497
2498 // Following passes run at the end of the pre-RA pipeline, so the pass they
2499 // are anchored on depends on which scheduler is enabled.
2500 auto insertAtEndOfPreRA = [this](auto &&Pass) {
2501 if (Opt.EnableSSAMachineScheduler)
2502 insertPass<RenameIndependentSubregsPass>(std::move(Pass));
2503 else
2504 insertPass<MachineSchedulerPass>(std::move(Pass));
2505 };
2506
2507 if (isPassEnabled(Opt: CLOpts.enable_pre_ra_optimizations))
2508 insertAtEndOfPreRA(GCNPreRAOptimizationsPass());
2509
2510 // Allow the scheduler to run before SIWholeQuadMode inserts exec manipulation
2511 // instructions that cause scheduling barriers.
2512 insertAtEndOfPreRA(SIWholeQuadModePass());
2513
2514 if (CLOpts.opt_exec_mask_pre_ra)
2515 insertAtEndOfPreRA(SIOptimizeExecMaskingPreRAPass());
2516
2517 // This is not an essential optimization and it has a noticeable impact on
2518 // compilation time, so we only enable it from O2.
2519 if (TM.getOptLevel() > CodeGenOptLevel::Less)
2520 insertAtEndOfPreRA(SIFormMemoryClausesPass());
2521
2522 return Base::addOptimizedRegAlloc(PMW);
2523}
2524
2525void AMDGPUCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
2526 if (getOptLevel() != CodeGenOptLevel::None)
2527 addMachineFunctionPass(Pass: AMDGPUPrepareAGPRAllocPass(), PMW);
2528}
2529
2530Expected<bool> AMDGPUCodeGenPassBuilder::addRegAssignAndRewriteOptimized(
2531 PassManagerWrapper &PMW) {
2532 if (auto Err = validateRegAllocOptions())
2533 return Err;
2534
2535 addMachineFunctionPass(Pass: GCNPreRALongBranchRegPass(), PMW);
2536
2537 // SGPR allocation - default to greedy at -O1 and above.
2538 if (SGPRRegAllocNPM == RegAllocType::Fast)
2539 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateSGPRs, "sgpr", false}),
2540 PMW);
2541 else
2542 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateSGPRs, "sgpr"}), PMW);
2543
2544 // Commit allocated register changes. This is mostly necessary because too
2545 // many things rely on the use lists of the physical registers, such as the
2546 // verifier. This is only necessary with allocators which use LiveIntervals,
2547 // since FastRegAlloc does the replacements itself.
2548 addMachineFunctionPass(Pass: VirtRegRewriterPass(false), PMW);
2549
2550 // At this point, the sgpr-regalloc has been done and it is good to have the
2551 // stack slot coloring to try to optimize the SGPR spill stack indices before
2552 // attempting the custom SGPR spill lowering.
2553 addMachineFunctionPass(Pass: StackSlotColoringPass(), PMW);
2554
2555 // Equivalent of PEI for SGPRs.
2556 addMachineFunctionPass(Pass: SILowerSGPRSpillsPass(), PMW);
2557
2558 // To Allocate wwm registers used in whole quad mode operations (for shaders).
2559 addMachineFunctionPass(Pass: SIPreAllocateWWMRegsPass(), PMW);
2560
2561 // WWM allocation - default to greedy at -O1 and above.
2562 if (WWMRegAllocNPM == RegAllocType::Fast)
2563 addMachineFunctionPass(
2564 Pass: RegAllocFastPass({onlyAllocateWWMRegs, "wwm", false}), PMW);
2565 else
2566 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateWWMRegs, "wwm"}), PMW);
2567 addMachineFunctionPass(Pass: SILowerWWMCopiesPass(), PMW);
2568 addMachineFunctionPass(Pass: VirtRegRewriterPass(false), PMW);
2569 addMachineFunctionPass(Pass: AMDGPUReserveWWMRegsPass(), PMW);
2570
2571 // VGPR allocation - default to greedy at -O1 and above.
2572 if (VGPRRegAllocNPM == RegAllocType::Fast)
2573 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2574 else
2575 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2576
2577 addPreRewrite(PMW);
2578 addMachineFunctionPass(Pass: VirtRegRewriterPass(true), PMW);
2579
2580 addMachineFunctionPass(Pass: AMDGPUMarkLastScratchLoadPass(), PMW);
2581 return true;
2582}
2583
2584void AMDGPUCodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) {
2585 addMachineFunctionPass(Pass: SIFixVGPRCopiesPass(), PMW);
2586 if (TM.getOptLevel() > CodeGenOptLevel::None)
2587 addMachineFunctionPass(Pass: SIOptimizeExecMaskingPass(), PMW);
2588 Base::addPostRegAlloc(PMW);
2589}
2590
2591void AMDGPUCodeGenPassBuilder::addPreSched2(PassManagerWrapper &PMW) {
2592 if (TM.getOptLevel() > CodeGenOptLevel::None)
2593 addMachineFunctionPass(Pass: SIShrinkInstructionsPass(), PMW);
2594 addMachineFunctionPass(Pass: SIPostRABundlerPass(), PMW);
2595}
2596
2597void AMDGPUCodeGenPassBuilder::addPostBBSections(PassManagerWrapper &PMW) {
2598 // We run this later to avoid passes like livedebugvalues and BBSections
2599 // having to deal with the apparent multi-entry functions we may generate.
2600 addMachineFunctionPass(Pass: AMDGPUPreloadKernArgPrologPass(), PMW);
2601}
2602
2603void AMDGPUCodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
2604 if (isPassEnabled(Opt: CLOpts.enable_vopd, Level: CodeGenOptLevel::Less)) {
2605 addMachineFunctionPass(Pass: GCNCreateVOPDPass(), PMW);
2606 }
2607
2608 addMachineFunctionPass(Pass: SIMemoryLegalizerPass(), PMW);
2609 if (TM.getOptLevel() > CodeGenOptLevel::None)
2610 addMachineFunctionPass(Pass: SIPostRA16BitMovFoldingPass(), PMW);
2611 if (mayNeedAsyncBarrierArriveWorkaround(TT: TM.getTargetTriple()))
2612 addMachineFunctionPass(Pass: SIInsertAsyncBarrierArriveWorkaroundPass(), PMW);
2613 addMachineFunctionPass(Pass: SIInsertWaitcntsPass(), PMW);
2614
2615 addMachineFunctionPass(Pass: SIModeRegisterPass(), PMW);
2616
2617 if (TM.getOptLevel() > CodeGenOptLevel::None)
2618 addMachineFunctionPass(Pass: SIInsertHardClausesPass(), PMW);
2619
2620 addMachineFunctionPass(Pass: SILateBranchLoweringPass(), PMW);
2621
2622 if (CLOpts.set_wave_priority)
2623 addMachineFunctionPass(Pass: AMDGPUSetWavePriorityPass(), PMW);
2624
2625 if (TM.getOptLevel() > CodeGenOptLevel::None)
2626 addMachineFunctionPass(Pass: SIPreEmitPeepholePass(), PMW);
2627
2628 // The hazard recognizer that runs as part of the post-ra scheduler does not
2629 // guarantee to be able handle all hazards correctly. This is because if there
2630 // are multiple scheduling regions in a basic block, the regions are scheduled
2631 // bottom up, so when we begin to schedule a region we don't know what
2632 // instructions were emitted directly before it.
2633 //
2634 // Here we add a stand-alone hazard recognizer pass which can handle all
2635 // cases.
2636 addMachineFunctionPass(Pass: PostRAHazardRecognizerPass(), PMW);
2637 addMachineFunctionPass(Pass: AMDGPUWaitSGPRHazardsPass(), PMW);
2638 addMachineFunctionPass(Pass: AMDGPULowerVGPREncodingPass(), PMW);
2639
2640 if (isPassEnabled(Opt: CLOpts.enable_delay_alu, Level: CodeGenOptLevel::Less)) {
2641 addMachineFunctionPass(Pass: AMDGPUInsertDelayAluPass(), PMW);
2642 }
2643
2644 addMachineFunctionPass(Pass: BranchRelaxationPass(), PMW);
2645}
2646
2647bool AMDGPUCodeGenPassBuilder::isPassEnabled(BoolOrDefault Opt,
2648 CodeGenOptLevel Level) const {
2649 return valueOr(X: Opt, Default: TM.getOptLevel() >= Level);
2650}
2651
2652void AMDGPUCodeGenPassBuilder::addEarlyCSEOrGVNPass(PassManagerWrapper &PMW) {
2653 if (TM.getOptLevel() == CodeGenOptLevel::Aggressive)
2654 addFunctionPass(Pass: GVNPass(), PMW);
2655 else
2656 addFunctionPass(Pass: EarlyCSEPass(), PMW);
2657}
2658
2659void AMDGPUCodeGenPassBuilder::addStraightLineScalarOptimizationPasses(
2660 PassManagerWrapper &PMW) {
2661 if (CLOpts.loop_prefetch)
2662 addFunctionPass(Pass: LoopDataPrefetchPass(), PMW);
2663
2664 addFunctionPass(Pass: SeparateConstOffsetFromGEPPass(), PMW);
2665
2666 // ReassociateGEPs exposes more opportunities for SLSR. See
2667 // the example in reassociate-geps-and-slsr.ll.
2668 addFunctionPass(Pass: StraightLineStrengthReducePass(), PMW);
2669
2670 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or
2671 // EarlyCSE can reuse.
2672 addEarlyCSEOrGVNPass(PMW);
2673
2674 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
2675 addFunctionPass(Pass: NaryReassociatePass(), PMW);
2676
2677 // NaryReassociate on GEPs creates redundant common expressions, so run
2678 // EarlyCSE after it.
2679 addFunctionPass(Pass: EarlyCSEPass(), PMW);
2680}
2681