1//===-- AMDGPUTargetMachine.cpp - TargetMachine for hw codegen targets-----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file contains both AMDGPU target machine and the CodeGen pass builder.
11/// The AMDGPU target machine contains all of the hardware specific information
12/// needed to emit code for SI+ GPUs in the legacy pass manager pipeline. The
13/// CodeGen pass builder handles the pass pipeline for new pass manager.
14//
15//===----------------------------------------------------------------------===//
16
17#include "AMDGPUTargetMachine.h"
18#include "AMDGPU.h"
19#include "AMDGPUAliasAnalysis.h"
20#include "AMDGPUAsmPrinter.h"
21#include "AMDGPUBarrierLatency.h"
22#include "AMDGPUCoExecSchedStrategy.h"
23#include "AMDGPUCtorDtorLowering.h"
24#include "AMDGPUExportClustering.h"
25#include "AMDGPUExportKernelRuntimeHandles.h"
26#include "AMDGPUHazardLatency.h"
27#include "AMDGPUIGroupLP.h"
28#include "AMDGPUISelDAGToDAG.h"
29#include "AMDGPULowerVGPREncoding.h"
30#include "AMDGPUMacroFusion.h"
31#include "AMDGPUNextUseAnalysis.h"
32#include "AMDGPUPerfHintAnalysis.h"
33#include "AMDGPUPreloadKernArgProlog.h"
34#include "AMDGPUPrepareAGPRAlloc.h"
35#include "AMDGPURemoveIncompatibleFunctions.h"
36#include "AMDGPUReserveWWMRegs.h"
37#include "AMDGPUResourceUsageAnalysis.h"
38#include "AMDGPUSplitModule.h"
39#include "AMDGPUTargetObjectFile.h"
40#include "AMDGPUTargetTransformInfo.h"
41#include "AMDGPUUnifyDivergentExitNodes.h"
42#include "AMDGPUWaitSGPRHazards.h"
43#include "GCNDPPCombine.h"
44#include "GCNIterativeScheduler.h"
45#include "GCNNSAReassign.h"
46#include "GCNPreRALongBranchReg.h"
47#include "GCNPreRAOptimizations.h"
48#include "GCNRewritePartialRegUses.h"
49#include "GCNSchedStrategy.h"
50#include "GCNVOPDUtils.h"
51#include "R600.h"
52#include "R600TargetMachine.h"
53#include "SIFixSGPRCopies.h"
54#include "SIFixVGPRCopies.h"
55#include "SIFoldOperands.h"
56#include "SIFormMemoryClauses.h"
57#include "SILoadStoreOptimizer.h"
58#include "SILowerControlFlow.h"
59#include "SILowerSGPRSpills.h"
60#include "SILowerWWMCopies.h"
61#include "SIMachineFunctionInfo.h"
62#include "SIMachineScheduler.h"
63#include "SIOptimizeExecMasking.h"
64#include "SIOptimizeExecMaskingPreRA.h"
65#include "SIOptimizeVGPRLiveRange.h"
66#include "SIPeepholeSDWA.h"
67#include "SIPostRABundler.h"
68#include "SIPreAllocateWWMRegs.h"
69#include "SIShrinkInstructions.h"
70#include "SIWholeQuadMode.h"
71#include "TargetInfo/AMDGPUTargetInfo.h"
72#include "Utils/AMDGPUBaseInfo.h"
73#include "llvm/Analysis/CGSCCPassManager.h"
74#include "llvm/Analysis/KernelInfo.h"
75#include "llvm/CodeGen/AtomicExpand.h"
76#include "llvm/CodeGen/BranchRelaxation.h"
77#include "llvm/CodeGen/DeadMachineInstructionElim.h"
78#include "llvm/CodeGen/DetectDeadLanes.h"
79#include "llvm/CodeGen/EarlyIfConversion.h"
80#include "llvm/CodeGen/FuncletLayout.h"
81#include "llvm/CodeGen/GCMetadata.h"
82#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
83#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
84#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
85#include "llvm/CodeGen/GlobalISel/Legalizer.h"
86#include "llvm/CodeGen/GlobalISel/Localizer.h"
87#include "llvm/CodeGen/MIRParser/MIParser.h"
88#include "llvm/CodeGen/MachineCSE.h"
89#include "llvm/CodeGen/MachineLICM.h"
90#include "llvm/CodeGen/MachinePipeliner.h"
91#include "llvm/CodeGen/MachineScheduler.h"
92#include "llvm/CodeGen/PHIElimination.h"
93#include "llvm/CodeGen/Passes.h"
94#include "llvm/CodeGen/PatchableFunction.h"
95#include "llvm/CodeGen/PostRAHazardRecognizer.h"
96#include "llvm/CodeGen/RegAllocGreedyPass.h"
97#include "llvm/CodeGen/RegAllocRegistry.h"
98#include "llvm/CodeGen/RenameIndependentSubregs.h"
99#include "llvm/CodeGen/ShadowStackGCLowering.h"
100#include "llvm/CodeGen/StackSlotColoring.h"
101#include "llvm/CodeGen/TargetPassConfig.h"
102#include "llvm/CodeGen/TwoAddressInstructionPass.h"
103#include "llvm/IR/DiagnosticInfo.h"
104#include "llvm/IR/IntrinsicsAMDGPU.h"
105#include "llvm/IR/Module.h"
106#include "llvm/IR/PassManager.h"
107#include "llvm/IR/PatternMatch.h"
108#include "llvm/InitializePasses.h"
109#include "llvm/MC/TargetRegistry.h"
110#include "llvm/Passes/CodeGenPassBuilder.h"
111#include "llvm/Passes/PassBuilder.h"
112#include "llvm/Support/Compiler.h"
113#include "llvm/Support/FormatVariadic.h"
114#include "llvm/TargetParser/AMDGPUTargetParser.h"
115#include "llvm/Transforms/HipStdPar/HipStdPar.h"
116#include "llvm/Transforms/IPO.h"
117#include "llvm/Transforms/IPO/AlwaysInliner.h"
118#include "llvm/Transforms/IPO/ExpandVariadics.h"
119#include "llvm/Transforms/IPO/GlobalDCE.h"
120#include "llvm/Transforms/IPO/Internalize.h"
121#include "llvm/Transforms/Scalar.h"
122#include "llvm/Transforms/Scalar/EarlyCSE.h"
123#include "llvm/Transforms/Scalar/FlattenCFG.h"
124#include "llvm/Transforms/Scalar/GVN.h"
125#include "llvm/Transforms/Scalar/InferAddressSpaces.h"
126#include "llvm/Transforms/Scalar/LICM.h"
127#include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
128#include "llvm/Transforms/Scalar/LoopPassManager.h"
129#include "llvm/Transforms/Scalar/NaryReassociate.h"
130#include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"
131#include "llvm/Transforms/Scalar/Sink.h"
132#include "llvm/Transforms/Scalar/StraightLineStrengthReduce.h"
133#include "llvm/Transforms/Scalar/StructurizeCFG.h"
134#include "llvm/Transforms/Utils.h"
135#include "llvm/Transforms/Utils/FixIrreducible.h"
136#include "llvm/Transforms/Utils/LCSSA.h"
137#include "llvm/Transforms/Utils/LowerSwitch.h"
138#include "llvm/Transforms/Utils/SimplifyLibCalls.h"
139#include "llvm/Transforms/Utils/UnifyLoopExits.h"
140#include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
141#include <optional>
142
143using namespace llvm;
144using namespace llvm::PatternMatch;
145
146namespace {
147//===----------------------------------------------------------------------===//
148// AMDGPU CodeGen Pass Builder interface.
149//===----------------------------------------------------------------------===//
150
151class AMDGPUCodeGenPassBuilder : public CodeGenPassBuilder {
152 using Base = CodeGenPassBuilder;
153
154 GCNTargetMachine &getTM() const {
155 return static_cast<GCNTargetMachine &>(TM);
156 }
157
158public:
159 AMDGPUCodeGenPassBuilder(GCNTargetMachine &TM,
160 const CGPassBuilderOption &Opts,
161 PassInstrumentationCallbacks *PIC);
162
163 void addIRPasses(PassManagerWrapper &PMW) override;
164 void addCodeGenPrepare(PassManagerWrapper &PMW) override;
165 void addPreISel(PassManagerWrapper &PMW) override;
166 void addILPOpts(PassManagerWrapper &PMW) override;
167 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
168 void addAsmPrinter(PassManagerWrapper &PMW) override;
169 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
170 Error addInstSelector(PassManagerWrapper &PMW) override;
171 Error addIRTranslator(PassManagerWrapper &PMW) override;
172 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
173 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
174 void addPreRegBankSelect(PassManagerWrapper &PMW) override;
175 Error addRegBankSelect(PassManagerWrapper &PMW) override;
176 void addPreGlobalInstructionSelect(PassManagerWrapper &PMW) override;
177 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
178 void addPreRewrite(PassManagerWrapper &PMW) override;
179 void addMachineSSAOptimization(PassManagerWrapper &PMW) override;
180 void addPostRegAlloc(PassManagerWrapper &PMW) override;
181 void addPreEmitPass(PassManagerWrapper &PMW) override;
182 Error addRegAssignAndRewriteFast(PassManagerWrapper &PMW) override;
183 Expected<bool>
184 addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) override;
185 void addPreRegAlloc(PassManagerWrapper &PMW) override;
186 Error addFastRegAlloc(PassManagerWrapper &PMW) override;
187 Error addOptimizedRegAlloc(PassManagerWrapper &PMW) override;
188 void addPreSched2(PassManagerWrapper &PMW) override;
189 void addPostBBSections(PassManagerWrapper &PMW) override;
190
191private:
192 Error validateRegAllocOptions() const;
193
194public:
195 /// Check if a pass is enabled given \p Opt option. The option always
196 /// overrides defaults if explicitly used. Otherwise its default will be used
197 /// given that a pass shall work at an optimization \p Level minimum.
198 bool isPassEnabled(const cl::opt<bool> &Opt,
199 CodeGenOptLevel Level = CodeGenOptLevel::Default) const;
200 void addEarlyCSEOrGVNPass(PassManagerWrapper &PMW);
201 void addStraightLineScalarOptimizationPasses(PassManagerWrapper &PMW);
202};
203
204class SGPRRegisterRegAlloc : public RegisterRegAllocBase<SGPRRegisterRegAlloc> {
205public:
206 SGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
207 : RegisterRegAllocBase(N, D, C) {}
208};
209
210class VGPRRegisterRegAlloc : public RegisterRegAllocBase<VGPRRegisterRegAlloc> {
211public:
212 VGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
213 : RegisterRegAllocBase(N, D, C) {}
214};
215
216class WWMRegisterRegAlloc : public RegisterRegAllocBase<WWMRegisterRegAlloc> {
217public:
218 WWMRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
219 : RegisterRegAllocBase(N, D, C) {}
220};
221
222static bool onlyAllocateSGPRs(const TargetRegisterInfo &TRI,
223 const MachineRegisterInfo &MRI,
224 const Register Reg) {
225 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
226 return static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC);
227}
228
229static bool onlyAllocateVGPRs(const TargetRegisterInfo &TRI,
230 const MachineRegisterInfo &MRI,
231 const Register Reg) {
232 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
233 return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC);
234}
235
236static bool onlyAllocateWWMRegs(const TargetRegisterInfo &TRI,
237 const MachineRegisterInfo &MRI,
238 const Register Reg) {
239 const SIMachineFunctionInfo *MFI =
240 MRI.getMF().getInfo<SIMachineFunctionInfo>();
241 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
242 return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC) &&
243 MFI->checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG);
244}
245
246/// -{sgpr|wwm|vgpr}-regalloc=... command line option.
247static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
248
249/// A dummy default pass factory indicates whether the register allocator is
250/// overridden on the command line.
251static llvm::once_flag InitializeDefaultSGPRRegisterAllocatorFlag;
252static llvm::once_flag InitializeDefaultVGPRRegisterAllocatorFlag;
253static llvm::once_flag InitializeDefaultWWMRegisterAllocatorFlag;
254
255static SGPRRegisterRegAlloc
256defaultSGPRRegAlloc("default",
257 "pick SGPR register allocator based on -O option",
258 useDefaultRegisterAllocator);
259
260static cl::opt<SGPRRegisterRegAlloc::FunctionPassCtor, false,
261 RegisterPassParser<SGPRRegisterRegAlloc>>
262SGPRRegAlloc("sgpr-regalloc", cl::Hidden, cl::init(Val: &useDefaultRegisterAllocator),
263 cl::desc("Register allocator to use for SGPRs"));
264
265static cl::opt<VGPRRegisterRegAlloc::FunctionPassCtor, false,
266 RegisterPassParser<VGPRRegisterRegAlloc>>
267VGPRRegAlloc("vgpr-regalloc", cl::Hidden, cl::init(Val: &useDefaultRegisterAllocator),
268 cl::desc("Register allocator to use for VGPRs"));
269
270static cl::opt<WWMRegisterRegAlloc::FunctionPassCtor, false,
271 RegisterPassParser<WWMRegisterRegAlloc>>
272 WWMRegAlloc("wwm-regalloc", cl::Hidden,
273 cl::init(Val: &useDefaultRegisterAllocator),
274 cl::desc("Register allocator to use for WWM registers"));
275
276// New pass manager register allocator options for AMDGPU
277static cl::opt<RegAllocType, false, RegAllocTypeParser> SGPRRegAllocNPM(
278 "sgpr-regalloc-npm", cl::Hidden, cl::init(Val: RegAllocType::Default),
279 cl::desc("Register allocator for SGPRs (new pass manager)"));
280
281static cl::opt<RegAllocType, false, RegAllocTypeParser> VGPRRegAllocNPM(
282 "vgpr-regalloc-npm", cl::Hidden, cl::init(Val: RegAllocType::Default),
283 cl::desc("Register allocator for VGPRs (new pass manager)"));
284
285static cl::opt<RegAllocType, false, RegAllocTypeParser> WWMRegAllocNPM(
286 "wwm-regalloc-npm", cl::Hidden, cl::init(Val: RegAllocType::Default),
287 cl::desc("Register allocator for WWM registers (new pass manager)"));
288
289/// Check if the given RegAllocType is supported for AMDGPU NPM register
290/// allocation. Only Fast and Greedy are supported; Basic and PBQP are not.
291static Error checkRegAllocSupported(RegAllocType RAType, StringRef RegName) {
292 if (RAType == RegAllocType::Basic || RAType == RegAllocType::PBQP) {
293 return make_error<StringError>(
294 Args: Twine("unsupported register allocator '") +
295 (RAType == RegAllocType::Basic ? "basic" : "pbqp") + "' for " +
296 RegName + " registers",
297 Args: inconvertibleErrorCode());
298 }
299 return Error::success();
300}
301
302Error AMDGPUCodeGenPassBuilder::validateRegAllocOptions() const {
303 // 1. Generic --regalloc-npm is not supported for AMDGPU.
304 if (Opt.RegAlloc != RegAllocType::Unset) {
305 return make_error<StringError>(
306 Args: "-regalloc-npm not supported for amdgcn. Use -sgpr-regalloc-npm, "
307 "-vgpr-regalloc-npm, and -wwm-regalloc-npm",
308 Args: inconvertibleErrorCode());
309 }
310
311 // 2. Legacy PM regalloc options are not compatible with NPM.
312 if (SGPRRegAlloc.getNumOccurrences() > 0 ||
313 VGPRRegAlloc.getNumOccurrences() > 0 ||
314 WWMRegAlloc.getNumOccurrences() > 0) {
315 return make_error<StringError>(
316 Args: "-sgpr-regalloc, -vgpr-regalloc, and -wwm-regalloc are legacy PM "
317 "options. Use -sgpr-regalloc-npm, -vgpr-regalloc-npm, and "
318 "-wwm-regalloc-npm with the new pass manager",
319 Args: inconvertibleErrorCode());
320 }
321
322 // 3. Only Fast and Greedy allocators are supported for AMDGPU.
323 if (auto Err = checkRegAllocSupported(RAType: SGPRRegAllocNPM, RegName: "SGPR"))
324 return Err;
325 if (auto Err = checkRegAllocSupported(RAType: WWMRegAllocNPM, RegName: "WWM"))
326 return Err;
327 if (auto Err = checkRegAllocSupported(RAType: VGPRRegAllocNPM, RegName: "VGPR"))
328 return Err;
329
330 return Error::success();
331}
332
333static void initializeDefaultSGPRRegisterAllocatorOnce() {
334 RegisterRegAlloc::FunctionPassCtor Ctor = SGPRRegisterRegAlloc::getDefault();
335
336 if (!Ctor) {
337 Ctor = SGPRRegAlloc;
338 SGPRRegisterRegAlloc::setDefault(SGPRRegAlloc);
339 }
340}
341
342static void initializeDefaultVGPRRegisterAllocatorOnce() {
343 RegisterRegAlloc::FunctionPassCtor Ctor = VGPRRegisterRegAlloc::getDefault();
344
345 if (!Ctor) {
346 Ctor = VGPRRegAlloc;
347 VGPRRegisterRegAlloc::setDefault(VGPRRegAlloc);
348 }
349}
350
351static void initializeDefaultWWMRegisterAllocatorOnce() {
352 RegisterRegAlloc::FunctionPassCtor Ctor = WWMRegisterRegAlloc::getDefault();
353
354 if (!Ctor) {
355 Ctor = WWMRegAlloc;
356 WWMRegisterRegAlloc::setDefault(WWMRegAlloc);
357 }
358}
359
360static FunctionPass *createBasicSGPRRegisterAllocator() {
361 return createBasicRegisterAllocator(F: onlyAllocateSGPRs);
362}
363
364static FunctionPass *createGreedySGPRRegisterAllocator() {
365 return createGreedyRegisterAllocator(F: onlyAllocateSGPRs);
366}
367
368static FunctionPass *createFastSGPRRegisterAllocator() {
369 return createFastRegisterAllocator(F: onlyAllocateSGPRs, ClearVirtRegs: false);
370}
371
372static FunctionPass *createBasicVGPRRegisterAllocator() {
373 return createBasicRegisterAllocator(F: onlyAllocateVGPRs);
374}
375
376static FunctionPass *createGreedyVGPRRegisterAllocator() {
377 return createGreedyRegisterAllocator(F: onlyAllocateVGPRs);
378}
379
380static FunctionPass *createFastVGPRRegisterAllocator() {
381 return createFastRegisterAllocator(F: onlyAllocateVGPRs, ClearVirtRegs: true);
382}
383
384static FunctionPass *createBasicWWMRegisterAllocator() {
385 return createBasicRegisterAllocator(F: onlyAllocateWWMRegs);
386}
387
388static FunctionPass *createGreedyWWMRegisterAllocator() {
389 return createGreedyRegisterAllocator(F: onlyAllocateWWMRegs);
390}
391
392static FunctionPass *createFastWWMRegisterAllocator() {
393 return createFastRegisterAllocator(F: onlyAllocateWWMRegs, ClearVirtRegs: false);
394}
395
396static SGPRRegisterRegAlloc basicRegAllocSGPR(
397 "basic", "basic register allocator", createBasicSGPRRegisterAllocator);
398static SGPRRegisterRegAlloc greedyRegAllocSGPR(
399 "greedy", "greedy register allocator", createGreedySGPRRegisterAllocator);
400
401static SGPRRegisterRegAlloc fastRegAllocSGPR(
402 "fast", "fast register allocator", createFastSGPRRegisterAllocator);
403
404
405static VGPRRegisterRegAlloc basicRegAllocVGPR(
406 "basic", "basic register allocator", createBasicVGPRRegisterAllocator);
407static VGPRRegisterRegAlloc greedyRegAllocVGPR(
408 "greedy", "greedy register allocator", createGreedyVGPRRegisterAllocator);
409
410static VGPRRegisterRegAlloc fastRegAllocVGPR(
411 "fast", "fast register allocator", createFastVGPRRegisterAllocator);
412static WWMRegisterRegAlloc basicRegAllocWWMReg("basic",
413 "basic register allocator",
414 createBasicWWMRegisterAllocator);
415static WWMRegisterRegAlloc
416 greedyRegAllocWWMReg("greedy", "greedy register allocator",
417 createGreedyWWMRegisterAllocator);
418static WWMRegisterRegAlloc fastRegAllocWWMReg("fast", "fast register allocator",
419 createFastWWMRegisterAllocator);
420
421static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {
422 return Phase == ThinOrFullLTOPhase::FullLTOPreLink ||
423 Phase == ThinOrFullLTOPhase::ThinLTOPreLink;
424}
425} // anonymous namespace
426
427static cl::opt<bool>
428EnableEarlyIfConversion("amdgpu-early-ifcvt", cl::Hidden,
429 cl::desc("Run early if-conversion"),
430 cl::init(Val: false));
431
432static cl::opt<bool>
433OptExecMaskPreRA("amdgpu-opt-exec-mask-pre-ra", cl::Hidden,
434 cl::desc("Run pre-RA exec mask optimizations"),
435 cl::init(Val: true));
436
437static cl::opt<bool>
438 LowerCtorDtor("amdgpu-lower-global-ctor-dtor",
439 cl::desc("Lower GPU ctor / dtors to globals on the device."),
440 cl::init(Val: true), cl::Hidden);
441
442// Option to disable vectorizer for tests.
443static cl::opt<bool> EnableLoadStoreVectorizer(
444 "amdgpu-load-store-vectorizer",
445 cl::desc("Enable load store vectorizer"),
446 cl::init(Val: true),
447 cl::Hidden);
448
449// Option to run internalize pass.
450static cl::opt<bool> InternalizeSymbols(
451 "amdgpu-internalize-symbols",
452 cl::desc("Enable elimination of non-kernel functions and unused globals"),
453 cl::init(Val: false),
454 cl::Hidden);
455
456// Option to inline all early.
457static cl::opt<bool> EarlyInlineAll(
458 "amdgpu-early-inline-all",
459 cl::desc("Inline all functions early"),
460 cl::init(Val: false),
461 cl::Hidden);
462
463static cl::opt<bool> RemoveIncompatibleFunctions(
464 "amdgpu-enable-remove-incompatible-functions", cl::Hidden,
465 cl::desc("Enable removal of functions when they"
466 "use features not supported by the target GPU"),
467 cl::init(Val: true));
468
469static cl::opt<bool> EnableSDWAPeephole(
470 "amdgpu-sdwa-peephole",
471 cl::desc("Enable SDWA peepholer"),
472 cl::init(Val: true));
473
474static cl::opt<bool> EnableDPPCombine(
475 "amdgpu-dpp-combine",
476 cl::desc("Enable DPP combiner"),
477 cl::init(Val: true));
478
479// Enable address space based alias analysis
480static cl::opt<bool> EnableAMDGPUAliasAnalysis("enable-amdgpu-aa", cl::Hidden,
481 cl::desc("Enable AMDGPU Alias Analysis"),
482 cl::init(Val: true));
483
484static cl::opt<bool>
485 XnackSetting("amdgpu-xnack",
486 cl::desc("Force amdgpu.xnack value for testing"),
487 cl::ReallyHidden);
488
489static cl::opt<bool>
490 SramEccSetting("amdgpu-sramecc",
491 cl::desc("Force amdgpu.sramecc for testing"),
492 cl::ReallyHidden);
493
494// Enable lib calls simplifications
495static cl::opt<bool> EnableLibCallSimplify(
496 "amdgpu-simplify-libcall",
497 cl::desc("Enable amdgpu library simplifications"),
498 cl::init(Val: true),
499 cl::Hidden);
500
501static cl::opt<bool> EnableLowerKernelArguments(
502 "amdgpu-ir-lower-kernel-arguments",
503 cl::desc("Lower kernel argument loads in IR pass"),
504 cl::init(Val: true),
505 cl::Hidden);
506
507static cl::opt<bool> EnableRegReassign(
508 "amdgpu-reassign-regs",
509 cl::desc("Enable register reassign optimizations on gfx10+"),
510 cl::init(Val: true),
511 cl::Hidden);
512
513static cl::opt<bool> OptVGPRLiveRange(
514 "amdgpu-opt-vgpr-liverange",
515 cl::desc("Enable VGPR liverange optimizations for if-else structure"),
516 cl::init(Val: true), cl::Hidden);
517
518static cl::opt<ScanOptions> AMDGPUAtomicOptimizerStrategy(
519 "amdgpu-atomic-optimizer-strategy",
520 cl::desc("Select DPP or Iterative strategy for scan"),
521 cl::init(Val: ScanOptions::Iterative),
522 cl::values(
523 clEnumValN(ScanOptions::DPP, "DPP", "Use DPP operations for scan"),
524 clEnumValN(ScanOptions::Iterative, "Iterative",
525 "Use Iterative approach for scan"),
526 clEnumValN(ScanOptions::None, "None", "Disable atomic optimizer")));
527
528// Enable Mode register optimization
529static cl::opt<bool> EnableSIModeRegisterPass(
530 "amdgpu-mode-register",
531 cl::desc("Enable mode register pass"),
532 cl::init(Val: true),
533 cl::Hidden);
534
535// Enable GFX11+ s_delay_alu insertion
536static cl::opt<bool>
537 EnableInsertDelayAlu("amdgpu-enable-delay-alu",
538 cl::desc("Enable s_delay_alu insertion"),
539 cl::init(Val: true), cl::Hidden);
540
541// Enable GFX11+ VOPD
542static cl::opt<bool>
543 EnableVOPD("amdgpu-enable-vopd",
544 cl::desc("Enable VOPD, dual issue of VALU in wave32"),
545 cl::init(Val: true), cl::Hidden);
546
547// Option is used in lit tests to prevent deadcoding of patterns inspected.
548static cl::opt<bool>
549EnableDCEInRA("amdgpu-dce-in-ra",
550 cl::init(Val: true), cl::Hidden,
551 cl::desc("Enable machine DCE inside regalloc"));
552
553static cl::opt<bool> EnableSetWavePriority("amdgpu-set-wave-priority",
554 cl::desc("Adjust wave priority"),
555 cl::init(Val: false), cl::Hidden);
556
557static cl::opt<bool> EnableScalarIRPasses(
558 "amdgpu-scalar-ir-passes",
559 cl::desc("Enable scalar IR passes"),
560 cl::init(Val: true),
561 cl::Hidden);
562
563static cl::opt<bool> EnableLowerExecSync(
564 "amdgpu-enable-lower-exec-sync",
565 cl::desc("Enable lowering of execution synchronization."), cl::init(Val: true),
566 cl::Hidden);
567
568static cl::opt<bool>
569 EnableSwLowerLDS("amdgpu-enable-sw-lower-lds",
570 cl::desc("Enable lowering of lds to global memory pass "
571 "and asan instrument resulting IR."),
572 cl::init(Val: true), cl::Hidden);
573
574static cl::opt<bool, true> EnableObjectLinking(
575 "amdgpu-enable-object-linking",
576 cl::desc("Enable object linking for cross-TU LDS and ABI support"),
577 cl::location(L&: AMDGPUTargetMachine::EnableObjectLinking), cl::init(Val: false),
578 cl::Hidden);
579
580static cl::opt<bool, true> EnableLowerModuleLDS(
581 "amdgpu-enable-lower-module-lds", cl::desc("Enable lower module lds pass"),
582 cl::location(L&: AMDGPUTargetMachine::EnableLowerModuleLDS), cl::init(Val: true),
583 cl::Hidden);
584
585static cl::opt<bool> EnablePreRAOptimizations(
586 "amdgpu-enable-pre-ra-optimizations",
587 cl::desc("Enable Pre-RA optimizations pass"), cl::init(Val: true),
588 cl::Hidden);
589
590static cl::opt<bool> EnablePromoteKernelArguments(
591 "amdgpu-enable-promote-kernel-arguments",
592 cl::desc("Enable promotion of flat kernel pointer arguments to global"),
593 cl::Hidden, cl::init(Val: true));
594
595static cl::opt<bool> EnableImageIntrinsicOptimizer(
596 "amdgpu-enable-image-intrinsic-optimizer",
597 cl::desc("Enable image intrinsic optimizer pass"), cl::init(Val: true),
598 cl::Hidden);
599
600static cl::opt<bool>
601 EnableLoopPrefetch("amdgpu-loop-prefetch",
602 cl::desc("Enable loop data prefetch on AMDGPU"),
603 cl::Hidden, cl::init(Val: false));
604
605static cl::opt<std::string>
606 AMDGPUSchedStrategy("amdgpu-sched-strategy",
607 cl::desc("Select custom AMDGPU scheduling strategy."),
608 cl::Hidden, cl::init(Val: ""));
609
610// Scheduler selection is consulted both when creating the scheduler and from
611// overrideSchedPolicy(), so keep the attribute and global command line handling
612// in one helper.
613StringRef llvm::AMDGPU::getSchedStrategy(const Function &F) {
614 Attribute SchedStrategyAttr = F.getFnAttribute(Kind: "amdgpu-sched-strategy");
615 if (SchedStrategyAttr.isValid())
616 return SchedStrategyAttr.getValueAsString();
617
618 if (!AMDGPUSchedStrategy.empty())
619 return AMDGPUSchedStrategy;
620
621 return "";
622}
623
624static void
625diagnoseUnsupportedCoExecSchedulerSelection(const Function &F,
626 const GCNSubtarget &ST) {
627 if (ST.hasGFX1250Insts() || ST.hasGFX950Insts())
628 return;
629
630 F.getContext().diagnose(DI: DiagnosticInfoUnsupported(
631 F,
632 "'amdgpu-sched-strategy'='coexec' is only supported for gfx1250/gfx950",
633 DiagnosticLocation(), DS_Warning));
634}
635
636static bool useNoopPostScheduler(const Function &F) {
637 Attribute PostSchedStrategyAttr =
638 F.getFnAttribute(Kind: "amdgpu-post-sched-strategy");
639 return PostSchedStrategyAttr.isValid() &&
640 PostSchedStrategyAttr.getValueAsString() == "nop";
641}
642
643static cl::opt<bool> EnableRewritePartialRegUses(
644 "amdgpu-enable-rewrite-partial-reg-uses",
645 cl::desc("Enable rewrite partial reg uses pass"), cl::init(Val: true),
646 cl::Hidden);
647
648static cl::opt<bool> EnableHipStdPar(
649 "amdgpu-enable-hipstdpar",
650 cl::desc("Enable HIP Standard Parallelism Offload support"), cl::init(Val: false),
651 cl::Hidden);
652
653static cl::opt<bool>
654 EnableAMDGPUAttributor("amdgpu-attributor-enable",
655 cl::desc("Enable AMDGPUAttributorPass"),
656 cl::init(Val: true), cl::Hidden);
657
658static cl::opt<bool> HasClosedWorldAssumption(
659 "amdgpu-link-time-closed-world",
660 cl::desc("Whether has closed-world assumption at link time"),
661 cl::init(Val: false), cl::Hidden);
662
663static cl::opt<bool> EnableUniformIntrinsicCombine(
664 "amdgpu-enable-uniform-intrinsic-combine",
665 cl::desc("Enable/Disable the Uniform Intrinsic Combine Pass"),
666 cl::init(Val: true), cl::Hidden);
667
668static cl::opt<bool>
669 EnableMachinePipeliner("amdgpu-enable-pipeliner",
670 cl::desc("Enable Machine Pipeliner for AMDGCN"),
671 cl::init(Val: false), cl::Hidden);
672
673static cl::opt<bool>
674 UseSSAMachineScheduler("amdgpu-use-ssa-machine-scheduler",
675 cl::desc("Use the machine scheduler in SSA mode."),
676 cl::init(Val: false), cl::Hidden);
677
678extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
679 // Register the target
680 RegisterTargetMachine<R600TargetMachine> X(getTheR600Target());
681 RegisterTargetMachine<GCNTargetMachine> Y(getTheGCNTarget());
682 RegisterTargetMachine<GCNTargetMachine> YLegacy(getTheGCNLegacyTarget());
683
684 PassRegistry *PR = PassRegistry::getPassRegistry();
685 initializeR600ClauseMergePassPass(*PR);
686 initializeR600ControlFlowFinalizerPass(*PR);
687 initializeR600PacketizerPass(*PR);
688 initializeR600ExpandSpecialInstrsPassPass(*PR);
689 initializeR600VectorRegMergerPass(*PR);
690 initializeR600EmitClauseMarkersPass(*PR);
691 initializeR600MachineCFGStructurizerPass(*PR);
692 initializeGlobalISel(*PR);
693 initializeAMDGPUAsmPrinterPass(*PR);
694 initializeAMDGPUDAGToDAGISelLegacyPass(*PR);
695 initializeAMDGPUPrepareAGPRAllocLegacyPass(*PR);
696 initializeGCNDPPCombineLegacyPass(*PR);
697 initializeSILowerI1CopiesLegacyPass(*PR);
698 initializeAMDGPUGlobalISelDivergenceLoweringLegacyPass(*PR);
699 initializeAMDGPURegBankSelectLegacyPass(*PR);
700 initializeAMDGPURegBankLegalizeLegacyPass(*PR);
701 initializeSILowerWWMCopiesLegacyPass(*PR);
702 initializeAMDGPUMarkLastScratchLoadLegacyPass(*PR);
703 initializeSILowerSGPRSpillsLegacyPass(*PR);
704 initializeSIFixSGPRCopiesLegacyPass(*PR);
705 initializeSIFixVGPRCopiesLegacyPass(*PR);
706 initializeSIFoldOperandsLegacyPass(*PR);
707 initializeSIPeepholeSDWALegacyPass(*PR);
708 initializeSIShrinkInstructionsLegacyPass(*PR);
709 initializeSIOptimizeExecMaskingPreRALegacyPass(*PR);
710 initializeSIOptimizeVGPRLiveRangeLegacyPass(*PR);
711 initializeAMDGPUNextUseAnalysisLegacyPassPass(*PR);
712 initializeAMDGPUNextUseAnalysisPrinterLegacyPassPass(*PR);
713 initializeSILoadStoreOptimizerLegacyPass(*PR);
714 initializeAMDGPUCtorDtorLoweringLegacyPass(*PR);
715 initializeAMDGPUAlwaysInlinePass(*PR);
716 initializeAMDGPULowerExecSyncLegacyPass(*PR);
717 initializeAMDGPUSwLowerLDSLegacyPass(*PR);
718 initializeAMDGPUAnnotateUniformValuesLegacyPass(*PR);
719 initializeAMDGPUAtomicOptimizerPass(*PR);
720 initializeAMDGPULowerKernelArgumentsPass(*PR);
721 initializeAMDGPUPromoteKernelArgumentsPass(*PR);
722 initializeAMDGPULowerKernelAttributesPass(*PR);
723 initializeAMDGPUExportKernelRuntimeHandlesLegacyPass(*PR);
724 initializeAMDGPUPostLegalizerCombinerLegacyPass(*PR);
725 initializeAMDGPUPreLegalizerCombinerLegacyPass(*PR);
726 initializeAMDGPURegBankCombinerLegacyPass(*PR);
727 initializeAMDGPUPromoteAllocaPass(*PR);
728 initializeAMDGPUCodeGenPreparePass(*PR);
729 initializeAMDGPULateCodeGenPrepareLegacyPass(*PR);
730 initializeAMDGPURemoveIncompatibleFunctionsLegacyPass(*PR);
731 initializeAMDGPULowerModuleLDSLegacyPass(*PR);
732 initializeAMDGPULowerBufferFatPointersPass(*PR);
733 initializeAMDGPULowerIntrinsicsLegacyPass(*PR);
734 initializeAMDGPUReserveWWMRegsLegacyPass(*PR);
735 initializeAMDGPURewriteAGPRCopyMFMALegacyPass(*PR);
736 initializeAMDGPURewriteOutArgumentsPass(*PR);
737 initializeAMDGPURewriteUndefForPHILegacyPass(*PR);
738 initializeSIAnnotateControlFlowLegacyPass(*PR);
739 initializeAMDGPUInsertDelayAluLegacyPass(*PR);
740 initializeAMDGPULowerVGPREncodingLegacyPass(*PR);
741 initializeSIInsertHardClausesLegacyPass(*PR);
742 initializeSIInsertWaitcntsLegacyPass(*PR);
743 initializeSIModeRegisterLegacyPass(*PR);
744 initializeSIWholeQuadModeLegacyPass(*PR);
745 initializeSILowerControlFlowLegacyPass(*PR);
746 initializeSIPreEmitPeepholeLegacyPass(*PR);
747 initializeSILateBranchLoweringLegacyPass(*PR);
748 initializeSIMemoryLegalizerLegacyPass(*PR);
749 initializeSIOptimizeExecMaskingLegacyPass(*PR);
750 initializeSIPreAllocateWWMRegsLegacyPass(*PR);
751 initializeSIFormMemoryClausesLegacyPass(*PR);
752 initializeSIPostRABundlerLegacyPass(*PR);
753 initializeGCNCreateVOPDLegacyPass(*PR);
754 initializeAMDGPUUnifyDivergentExitNodesLegacyPass(*PR);
755 initializeAMDGPUAAWrapperPassPass(*PR);
756 initializeAMDGPUExternalAAWrapperPass(*PR);
757 initializeAMDGPUImageIntrinsicOptimizerPass(*PR);
758 initializeAMDGPUPrintfRuntimeBindingPass(*PR);
759 initializeAMDGPUResourceUsageAnalysisWrapperPassPass(*PR);
760 initializeGCNNSAReassignLegacyPass(*PR);
761 initializeGCNPreRAOptimizationsLegacyPass(*PR);
762 initializeGCNPreRALongBranchRegLegacyPass(*PR);
763 initializeGCNRewritePartialRegUsesLegacyPass(*PR);
764 initializeGCNRegPressurePrinterPass(*PR);
765 initializeAMDGPUPreloadKernArgPrologLegacyPass(*PR);
766 initializeAMDGPUWaitSGPRHazardsLegacyPass(*PR);
767 initializeAMDGPUPreloadKernelArgumentsLegacyPass(*PR);
768 initializeAMDGPUUniformIntrinsicCombineLegacyPass(*PR);
769 initializeSIPostRA16BitMovFoldingLegacyPass(*PR);
770}
771
772static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
773 return std::make_unique<AMDGPUTargetObjectFile>();
774}
775
776static ScheduleDAGInstrs *createSIMachineScheduler(MachineSchedContext *C) {
777 return new SIScheduleDAGMI(C);
778}
779
780static ScheduleDAGInstrs *
781createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
782 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
783 ScheduleDAGMILive *DAG =
784 new GCNScheduleDAGMILive(C, std::make_unique<GCNMaxOccupancySchedStrategy>(args&: C));
785 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII));
786 if (ST.shouldClusterStores())
787 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII));
788 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
789 DAG->addMutation(Mutation: createAMDGPUMacroFusionDAGMutation());
790 DAG->addMutation(Mutation: createAMDGPUExportClusteringDAGMutation());
791 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
792 DAG->addMutation(Mutation: createAMDGPUHazardLatencyDAGMutation(MF: C->MF));
793 return DAG;
794}
795
796static ScheduleDAGInstrs *
797createGCNMaxILPMachineScheduler(MachineSchedContext *C) {
798 ScheduleDAGMILive *DAG =
799 new GCNScheduleDAGMILive(C, std::make_unique<GCNMaxILPSchedStrategy>(args&: C));
800 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
801 return DAG;
802}
803
804static ScheduleDAGInstrs *
805createGCNMaxMemoryClauseMachineScheduler(MachineSchedContext *C) {
806 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
807 ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive(
808 C, std::make_unique<GCNMaxMemoryClauseSchedStrategy>(args&: C));
809 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII));
810 if (ST.shouldClusterStores())
811 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII));
812 DAG->addMutation(Mutation: createAMDGPUExportClusteringDAGMutation());
813 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
814 DAG->addMutation(Mutation: createAMDGPUHazardLatencyDAGMutation(MF: C->MF));
815 return DAG;
816}
817
818static ScheduleDAGInstrs *
819createIterativeGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
820 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
821 auto *DAG = new GCNIterativeScheduler(
822 C, GCNIterativeScheduler::SCHEDULE_LEGACYMAXOCCUPANCY);
823 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII));
824 if (ST.shouldClusterStores())
825 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII));
826 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
827 return DAG;
828}
829
830static ScheduleDAGInstrs *createMinRegScheduler(MachineSchedContext *C) {
831 auto *DAG = new GCNIterativeScheduler(
832 C, GCNIterativeScheduler::SCHEDULE_MINREGFORCED);
833 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
834 return DAG;
835}
836
837static ScheduleDAGInstrs *
838createIterativeILPMachineScheduler(MachineSchedContext *C) {
839 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
840 auto *DAG = new GCNIterativeScheduler(C, GCNIterativeScheduler::SCHEDULE_ILP);
841 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII));
842 if (ST.shouldClusterStores())
843 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII));
844 DAG->addMutation(Mutation: createAMDGPUMacroFusionDAGMutation());
845 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
846 return DAG;
847}
848
849static MachineSchedRegistry
850SISchedRegistry("si", "Run SI's custom scheduler",
851 createSIMachineScheduler);
852
853static MachineSchedRegistry
854GCNMaxOccupancySchedRegistry("gcn-max-occupancy",
855 "Run GCN scheduler to maximize occupancy",
856 createGCNMaxOccupancyMachineScheduler);
857
858static MachineSchedRegistry
859 GCNMaxILPSchedRegistry("gcn-max-ilp", "Run GCN scheduler to maximize ilp",
860 createGCNMaxILPMachineScheduler);
861
862static MachineSchedRegistry GCNMaxMemoryClauseSchedRegistry(
863 "gcn-max-memory-clause", "Run GCN scheduler to maximize memory clause",
864 createGCNMaxMemoryClauseMachineScheduler);
865
866static MachineSchedRegistry IterativeGCNMaxOccupancySchedRegistry(
867 "gcn-iterative-max-occupancy-experimental",
868 "Run GCN scheduler to maximize occupancy (experimental)",
869 createIterativeGCNMaxOccupancyMachineScheduler);
870
871static MachineSchedRegistry GCNMinRegSchedRegistry(
872 "gcn-iterative-minreg",
873 "Run GCN iterative scheduler for minimal register usage (experimental)",
874 createMinRegScheduler);
875
876static MachineSchedRegistry GCNILPSchedRegistry(
877 "gcn-iterative-ilp",
878 "Run GCN iterative scheduler for ILP scheduling (experimental)",
879 createIterativeILPMachineScheduler);
880
881LLVM_READNONE
882static StringRef getGPUOrDefault(const Triple &TT, StringRef GPU) {
883 if (!GPU.empty())
884 return GPU;
885
886 if (StringRef Name = AMDGPU::getArchNameFromSubArch(SubArch: TT.getSubArch());
887 !Name.empty())
888 return Name;
889
890 // Need to default to a target with flat support for HSA.
891 if (TT.isAMDGCN())
892 return TT.getOS() == Triple::AMDHSA ? "generic-hsa" : "generic";
893
894 return "r600";
895}
896
897static Reloc::Model getEffectiveRelocModel() {
898 // The AMDGPU toolchain only supports generating shared objects, so we
899 // must always use PIC.
900 return Reloc::PIC_;
901}
902
903AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT,
904 StringRef CPU, StringRef FS,
905 const TargetOptions &Options,
906 std::optional<Reloc::Model> RM,
907 std::optional<CodeModel::Model> CM,
908 CodeGenOptLevel OptLevel)
909 : CodeGenTargetMachineImpl(T, TT, getGPUOrDefault(TT, GPU: CPU), FS, Options,
910 getEffectiveRelocModel(),
911 getEffectiveCodeModel(CM, Default: CodeModel::Small),
912 OptLevel),
913 TLOF(createTLOF(TT: getTargetTriple())) {
914 initAsmInfo();
915 if (TT.isAMDGCN()) {
916 // Triple is missing a representation for non-empty, but unrecognized
917 // subarches. Only permit no subarch for any subtarget if it was really
918 // empty.
919 bool IsUnknownSubArch =
920 TT.getSubArch() == Triple::NoSubArch && TT.getArchName().size() != 6;
921 if (IsUnknownSubArch)
922 reportFatalUsageError(reason: "unknown subarch " + TT.getArchName());
923
924 if (TT.getSubArch() != Triple::NoSubArch) {
925 AMDGPU::GPUKind Kind = AMDGPU::parseArchAMDGCN(CPU);
926 Triple::SubArchType GPUSubArch = AMDGPU::getSubArch(AK: Kind);
927 if (Kind != AMDGPU::GK_NONE && GPUSubArch != TT.getSubArch() &&
928 TT.getSubArch() != AMDGPU::getMajorSubArch(SubArch: GPUSubArch)) {
929 reportFatalUsageError(reason: "invalid cpu '" + CPU + "' for subarch " +
930 TT.getArchName());
931 }
932 }
933
934 if (getMCSubtargetInfo().checkFeatures(FS: "+wavefrontsize64"))
935 MRI.reset(p: llvm::createGCNMCRegisterInfo(DwarfFlavour: AMDGPUDwarfFlavour::Wave64));
936 else if (getMCSubtargetInfo().checkFeatures(FS: "+wavefrontsize32"))
937 MRI.reset(p: llvm::createGCNMCRegisterInfo(DwarfFlavour: AMDGPUDwarfFlavour::Wave32));
938 }
939 LLT::setUseExtended(true);
940}
941
942bool AMDGPUTargetMachine::EnableFunctionCalls = false;
943bool AMDGPUTargetMachine::EnableObjectLinking = false;
944bool AMDGPUTargetMachine::EnableLowerModuleLDS = true;
945
946AMDGPUTargetMachine::~AMDGPUTargetMachine() = default;
947
948StringRef AMDGPUTargetMachine::getGPUName(const Function &F) const {
949 Attribute GPUAttr = F.getFnAttribute(Kind: "target-cpu");
950 return GPUAttr.isValid() ? GPUAttr.getValueAsString() : getTargetCPU();
951}
952
953StringRef AMDGPUTargetMachine::getFeatureString(const Function &F) const {
954 Attribute FSAttr = F.getFnAttribute(Kind: "target-features");
955
956 return FSAttr.isValid() ? FSAttr.getValueAsString()
957 : getTargetFeatureString();
958}
959
960llvm::ScheduleDAGInstrs *
961AMDGPUTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
962 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
963 ScheduleDAGMILive *DAG = createSchedLive(C);
964 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII));
965 if (ST.shouldClusterStores())
966 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII));
967 return DAG;
968}
969
970/// Predicate for Internalize pass.
971static bool mustPreserveGV(const GlobalValue &GV) {
972 if (const Function *F = dyn_cast<Function>(Val: &GV))
973 return F->isDeclaration() || F->getName().starts_with(Prefix: "__asan_") ||
974 F->getName().starts_with(Prefix: "__sanitizer_") ||
975 AMDGPU::isEntryFunctionCC(CC: F->getCallingConv());
976
977 GV.removeDeadConstantUsers();
978 return !GV.use_empty();
979}
980
981void AMDGPUTargetMachine::registerDefaultAliasAnalyses(AAManager &AAM) {
982 if (EnableAMDGPUAliasAnalysis)
983 AAM.registerFunctionAnalysis<AMDGPUAA>();
984}
985
986static Expected<ScanOptions>
987parseAMDGPUAtomicOptimizerStrategy(StringRef Params) {
988 if (Params.empty())
989 return ScanOptions::Iterative;
990 Params.consume_front(Prefix: "strategy=");
991 auto Result = StringSwitch<std::optional<ScanOptions>>(Params)
992 .Case(S: "dpp", Value: ScanOptions::DPP)
993 .Cases(CaseStrings: {"iterative", ""}, Value: ScanOptions::Iterative)
994 .Case(S: "none", Value: ScanOptions::None)
995 .Default(Value: std::nullopt);
996 if (Result)
997 return *Result;
998 return make_error<StringError>(Args: "invalid parameter", Args: inconvertibleErrorCode());
999}
1000
1001Expected<AMDGPUAttributorOptions>
1002parseAMDGPUAttributorPassOptions(StringRef Params) {
1003 AMDGPUAttributorOptions Result;
1004 while (!Params.empty()) {
1005 StringRef ParamName;
1006 std::tie(args&: ParamName, args&: Params) = Params.split(Separator: ';');
1007 if (ParamName == "closed-world") {
1008 Result.IsClosedWorld = true;
1009 } else {
1010 return make_error<StringError>(
1011 Args: formatv(Fmt: "invalid AMDGPUAttributor pass parameter '{0}' ", Vals&: ParamName)
1012 .str(),
1013 Args: inconvertibleErrorCode());
1014 }
1015 }
1016 return Result;
1017}
1018
1019void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
1020
1021#define GET_PASS_REGISTRY "AMDGPUPassRegistry.def"
1022#include "llvm/Passes/TargetPassRegistry.inc"
1023
1024 PB.registerPipelineParsingCallback(
1025 C: [this](StringRef Name, CGSCCPassManager &PM,
1026 ArrayRef<PassBuilder::PipelineElement> Pipeline) {
1027 if (Name == "amdgpu-attributor-cgscc" && getTargetTriple().isAMDGCN()) {
1028 PM.addPass(Pass: AMDGPUAttributorCGSCCPass(
1029 *static_cast<GCNTargetMachine *>(this)));
1030 return true;
1031 }
1032 return false;
1033 });
1034
1035 PB.registerScalarOptimizerLateEPCallback(
1036 C: [](FunctionPassManager &FPM, OptimizationLevel Level) {
1037 if (Level == OptimizationLevel::O0)
1038 return;
1039
1040 FPM.addPass(Pass: InferAddressSpacesPass());
1041 });
1042
1043 PB.registerVectorizerEndEPCallback(
1044 C: [](FunctionPassManager &FPM, OptimizationLevel Level) {
1045 if (Level == OptimizationLevel::O0)
1046 return;
1047
1048 FPM.addPass(Pass: InferAddressSpacesPass());
1049 });
1050
1051 PB.registerPipelineEarlySimplificationEPCallback(
1052 C: [this](ModulePassManager &PM, OptimizationLevel Level,
1053 ThinOrFullLTOPhase Phase) {
1054 if (!isLTOPreLink(Phase) && getTargetTriple().isAMDGCN()) {
1055 // When we are not using -fgpu-rdc, we can run accelerator code
1056 // selection relatively early, but still after linking to prevent
1057 // eager removal of potentially reachable symbols.
1058 if (EnableHipStdPar) {
1059 PM.addPass(Pass: HipStdParMathFixupPass());
1060 PM.addPass(Pass: HipStdParAcceleratorCodeSelectionPass());
1061 }
1062
1063 PM.addPass(Pass: AMDGPUPrintfRuntimeBindingPass());
1064 }
1065
1066 if (Level == OptimizationLevel::O0)
1067 return;
1068
1069 // We don't want to run internalization at per-module stage.
1070 if (InternalizeSymbols && !isLTOPreLink(Phase)) {
1071 PM.addPass(Pass: InternalizePass(mustPreserveGV));
1072 PM.addPass(Pass: GlobalDCEPass());
1073 }
1074
1075 if (EarlyInlineAll && !EnableFunctionCalls)
1076 PM.addPass(Pass: AMDGPUAlwaysInlinePass());
1077 });
1078
1079 PB.registerPeepholeEPCallback(
1080 C: [](FunctionPassManager &FPM, OptimizationLevel Level) {
1081 if (Level == OptimizationLevel::O0)
1082 return;
1083
1084 FPM.addPass(Pass: AMDGPUUseNativeCallsPass());
1085 if (EnableLibCallSimplify)
1086 FPM.addPass(Pass: AMDGPUSimplifyLibCallsPass());
1087
1088 if (EnableUniformIntrinsicCombine)
1089 FPM.addPass(Pass: AMDGPUUniformIntrinsicCombinePass());
1090 });
1091
1092 PB.registerCGSCCOptimizerLateEPCallback(
1093 C: [this](CGSCCPassManager &PM, OptimizationLevel Level) {
1094 if (Level == OptimizationLevel::O0)
1095 return;
1096
1097 FunctionPassManager FPM;
1098
1099 // Add promote kernel arguments pass to the opt pipeline right before
1100 // infer address spaces which is needed to do actual address space
1101 // rewriting.
1102 if (Level > OptimizationLevel::O1 && EnablePromoteKernelArguments)
1103 FPM.addPass(Pass: AMDGPUPromoteKernelArgumentsPass());
1104
1105 // Add infer address spaces pass to the opt pipeline after inlining
1106 // but before SROA to increase SROA opportunities.
1107 FPM.addPass(Pass: InferAddressSpacesPass());
1108
1109 // This should run after inlining to have any chance of doing
1110 // anything, and before other cleanup optimizations.
1111 FPM.addPass(Pass: AMDGPULowerKernelAttributesPass());
1112
1113 // Promote alloca to vector before SROA and loop unroll. If we
1114 // manage to eliminate allocas before unroll we may choose to unroll
1115 // less.
1116 FPM.addPass(Pass: AMDGPUPromoteAllocaToVectorPass(*this));
1117
1118 PM.addPass(Pass: createCGSCCToFunctionPassAdaptor(Pass: std::move(FPM)));
1119 });
1120
1121 // FIXME: Why is AMDGPUAttributor not in CGSCC?
1122 PB.registerOptimizerLastEPCallback(C: [this](ModulePassManager &MPM,
1123 OptimizationLevel Level,
1124 ThinOrFullLTOPhase Phase) {
1125 if (Level != OptimizationLevel::O0) {
1126 if (!isLTOPreLink(Phase)) {
1127 if (EnableAMDGPUAttributor && getTargetTriple().isAMDGCN()) {
1128 AMDGPUAttributorOptions Opts;
1129 MPM.addPass(Pass: AMDGPUAttributorPass(*this, Opts, Phase));
1130 }
1131 }
1132 }
1133 });
1134
1135 PB.registerFullLinkTimeOptimizationLastEPCallback(
1136 C: [this](ModulePassManager &PM, OptimizationLevel Level) {
1137 // Clean up redundant memory round-trips that the full-LTO pipeline,
1138 // unlike the non-LTO/ThinLTO ones, otherwise leaves for codegen.
1139 if (Level != OptimizationLevel::O0) {
1140 PM.addPass(Pass: createModuleToFunctionPassAdaptor(
1141 Pass: EarlyCSEPass(/*UseMemorySSA=*/true)));
1142 }
1143
1144 // When we are using -fgpu-rdc, we can only run accelerator code
1145 // selection after linking to prevent, otherwise we end up removing
1146 // potentially reachable symbols that were exported as external in other
1147 // modules.
1148 if (EnableHipStdPar) {
1149 PM.addPass(Pass: HipStdParMathFixupPass());
1150 PM.addPass(Pass: HipStdParAcceleratorCodeSelectionPass());
1151 }
1152 // We want to support the -lto-partitions=N option as "best effort".
1153 // For that, we need to lower LDS earlier in the pipeline before the
1154 // module is partitioned for codegen.
1155 if (EnableLowerExecSync)
1156 PM.addPass(Pass: AMDGPULowerExecSyncPass());
1157 if (EnableSwLowerLDS)
1158 PM.addPass(Pass: AMDGPUSwLowerLDSPass());
1159 if (EnableLowerModuleLDS)
1160 PM.addPass(Pass: AMDGPULowerModuleLDSPass(*this));
1161 if (Level != OptimizationLevel::O0) {
1162 // We only want to run this with O2 or higher since inliner and SROA
1163 // don't run in O1.
1164 if (Level != OptimizationLevel::O1) {
1165 PM.addPass(
1166 Pass: createModuleToFunctionPassAdaptor(Pass: InferAddressSpacesPass()));
1167 }
1168 // Do we really need internalization in LTO?
1169 if (InternalizeSymbols) {
1170 PM.addPass(Pass: InternalizePass(mustPreserveGV));
1171 PM.addPass(Pass: GlobalDCEPass());
1172 }
1173 if (EnableAMDGPUAttributor && getTargetTriple().isAMDGCN()) {
1174 AMDGPUAttributorOptions Opt;
1175 if (HasClosedWorldAssumption)
1176 Opt.IsClosedWorld = true;
1177 PM.addPass(Pass: AMDGPUAttributorPass(
1178 *this, Opt, ThinOrFullLTOPhase::FullLTOPostLink));
1179 }
1180 }
1181 if (!NoKernelInfoEndLTO) {
1182 FunctionPassManager FPM;
1183 FPM.addPass(Pass: KernelInfoPrinter(this));
1184 PM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM)));
1185 }
1186 });
1187
1188 PB.registerRegClassFilterParsingCallback(
1189 C: [](StringRef FilterName) -> RegAllocFilterFunc {
1190 if (FilterName == "sgpr")
1191 return onlyAllocateSGPRs;
1192 if (FilterName == "vgpr")
1193 return onlyAllocateVGPRs;
1194 if (FilterName == "wwm")
1195 return onlyAllocateWWMRegs;
1196 return nullptr;
1197 });
1198}
1199
1200bool AMDGPUTargetMachine::isNoopAddrSpaceCast(const DataLayout &DL,
1201 unsigned SrcAS,
1202 unsigned DestAS) const {
1203 return AMDGPU::isFlatGlobalAddrSpace(AS: SrcAS) &&
1204 AMDGPU::isFlatGlobalAddrSpace(AS: DestAS);
1205}
1206
1207unsigned AMDGPUTargetMachine::getAssumedAddrSpace(const Value *V) const {
1208 if (auto *Arg = dyn_cast<Argument>(Val: V);
1209 Arg &&
1210 AMDGPU::isModuleEntryFunctionCC(CC: Arg->getParent()->getCallingConv()) &&
1211 !Arg->hasByRefAttr())
1212 return AMDGPUAS::GLOBAL_ADDRESS;
1213
1214 const auto *LD = dyn_cast<LoadInst>(Val: V);
1215 if (!LD) // TODO: Handle invariant load like constant.
1216 return AMDGPUAS::UNKNOWN_ADDRESS_SPACE;
1217
1218 // It must be a generic pointer loaded.
1219 assert(V->getType()->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS);
1220
1221 const auto *Ptr = LD->getPointerOperand();
1222 if (Ptr->getType()->getPointerAddressSpace() != AMDGPUAS::CONSTANT_ADDRESS)
1223 return AMDGPUAS::UNKNOWN_ADDRESS_SPACE;
1224 // For a generic pointer loaded from the constant memory, it could be assumed
1225 // as a global pointer since the constant memory is only populated on the
1226 // host side. As implied by the offload programming model, only global
1227 // pointers could be referenced on the host side.
1228 return AMDGPUAS::GLOBAL_ADDRESS;
1229}
1230
1231std::pair<const Value *, unsigned>
1232AMDGPUTargetMachine::getPredicatedAddrSpace(const Value *V) const {
1233 if (auto *II = dyn_cast<IntrinsicInst>(Val: V)) {
1234 switch (II->getIntrinsicID()) {
1235 case Intrinsic::amdgcn_is_shared:
1236 return std::pair(II->getArgOperand(i: 0), AMDGPUAS::LOCAL_ADDRESS);
1237 case Intrinsic::amdgcn_is_private:
1238 return std::pair(II->getArgOperand(i: 0), AMDGPUAS::PRIVATE_ADDRESS);
1239 default:
1240 break;
1241 }
1242 return std::pair(nullptr, -1);
1243 }
1244 // Check the global pointer predication based on
1245 // (!is_share(p) && !is_private(p)). Note that logic 'and' is commutative and
1246 // the order of 'is_shared' and 'is_private' is not significant.
1247 Value *Ptr;
1248 if (match(
1249 V: const_cast<Value *>(V),
1250 P: m_c_And(L: m_Not(V: m_Intrinsic<Intrinsic::amdgcn_is_shared>(Ops: m_Value(V&: Ptr))),
1251 R: m_Not(V: m_Intrinsic<Intrinsic::amdgcn_is_private>(
1252 Ops: m_Deferred(V: Ptr))))))
1253 return std::pair(Ptr, AMDGPUAS::GLOBAL_ADDRESS);
1254
1255 return std::pair(nullptr, -1);
1256}
1257
1258unsigned
1259AMDGPUTargetMachine::getAddressSpaceForPseudoSourceKind(unsigned Kind) const {
1260 switch (Kind) {
1261 case PseudoSourceValue::Stack:
1262 case PseudoSourceValue::FixedStack:
1263 return AMDGPUAS::PRIVATE_ADDRESS;
1264 case PseudoSourceValue::ConstantPool:
1265 case PseudoSourceValue::GOT:
1266 case PseudoSourceValue::JumpTable:
1267 case PseudoSourceValue::GlobalValueCallEntry:
1268 case PseudoSourceValue::ExternalSymbolCallEntry:
1269 return AMDGPUAS::CONSTANT_ADDRESS;
1270 }
1271 return AMDGPUAS::FLAT_ADDRESS;
1272}
1273
1274bool AMDGPUTargetMachine::splitModule(
1275 Module &M, unsigned NumParts,
1276 function_ref<void(std::unique_ptr<Module> MPart)> ModuleCallback) {
1277 // FIXME(?): Would be better to use an already existing Analysis/PassManager,
1278 // but all current users of this API don't have one ready and would need to
1279 // create one anyway. Let's hide the boilerplate for now to keep it simple.
1280
1281 LoopAnalysisManager LAM;
1282 FunctionAnalysisManager FAM;
1283 CGSCCAnalysisManager CGAM;
1284 ModuleAnalysisManager MAM;
1285
1286 PassBuilder PB(this);
1287 PB.registerModuleAnalyses(MAM);
1288 PB.registerFunctionAnalyses(FAM);
1289 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
1290
1291 ModulePassManager MPM;
1292 MPM.addPass(Pass: AMDGPUSplitModulePass(NumParts, ModuleCallback));
1293 MPM.run(IR&: M, AM&: MAM);
1294 return true;
1295}
1296
1297//===----------------------------------------------------------------------===//
1298// GCN Target Machine (SI+)
1299//===----------------------------------------------------------------------===//
1300
1301GCNTargetMachine::GCNTargetMachine(const Target &T, const Triple &TT,
1302 StringRef CPU, StringRef FS,
1303 const TargetOptions &Options,
1304 std::optional<Reloc::Model> RM,
1305 std::optional<CodeModel::Model> CM,
1306 CodeGenOptLevel OL, bool JIT)
1307 : AMDGPUTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {
1308 setEnableDefaultMachineVerifier(false);
1309 // addFastRegAlloc inserts SIWholeQuadMode after TwoAddressInstructionPass.
1310 setEnableTiedFastRegAlloc(false);
1311}
1312
1313enum class OOBFlagValue {
1314 Any = 0,
1315 Relaxed = 1,
1316 Strict = 2,
1317};
1318
1319/// Returns the OOB mode encoded by a module flag.
1320/// An absent flag defaults to Any.
1321static OOBFlagValue getOOBFlagValue(const Module &M, StringRef FlagName) {
1322 const auto *Flag =
1323 mdconst::dyn_extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: FlagName));
1324 if (!Flag)
1325 return OOBFlagValue::Any;
1326 return static_cast<OOBFlagValue>(Flag->getZExtValue());
1327}
1328
1329/// Returns the xnack/sramecc setting encoded by a module flag.
1330/// Module flag values: 0 = disabled, 1 = enabled.
1331/// An absent flag defaults to Any.
1332AMDGPU::TargetIDSetting
1333GCNTargetMachine::getTargetIDSettingFromModuleFlag(const Module &M,
1334 StringRef FlagName) {
1335 using AMDGPU::TargetIDSetting;
1336
1337 if (XnackSetting.getNumOccurrences() > 0 && FlagName == "amdgpu.xnack")
1338 return XnackSetting ? TargetIDSetting::On : TargetIDSetting::Off;
1339 if (SramEccSetting.getNumOccurrences() > 0 && FlagName == "amdgpu.sramecc")
1340 return SramEccSetting ? TargetIDSetting::On : TargetIDSetting::Off;
1341
1342 const auto *Flag =
1343 mdconst::dyn_extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: FlagName));
1344 if (!Flag)
1345 return TargetIDSetting::Any;
1346 return Flag->getZExtValue() == 0 ? TargetIDSetting::Off : TargetIDSetting::On;
1347}
1348
1349const TargetSubtargetInfo *
1350GCNTargetMachine::getSubtargetImpl(const Function &F) const {
1351 StringRef GPU = getGPUName(F);
1352 StringRef FS = getFeatureString(F);
1353
1354 const Module &M = *F.getParent();
1355 OOBFlagValue BufOOB = getOOBFlagValue(M, FlagName: AMDGPUOOBMode::BufferFlag);
1356 OOBFlagValue TBufOOB = getOOBFlagValue(M, FlagName: AMDGPUOOBMode::TBufferFlag);
1357 bool BufRelaxed = BufOOB == OOBFlagValue::Relaxed;
1358 bool TBufRelaxed = TBufOOB == OOBFlagValue::Relaxed;
1359
1360 using AMDGPU::TargetIDSetting;
1361 TargetIDSetting Xnack = getTargetIDSettingFromModuleFlag(M, FlagName: "amdgpu.xnack");
1362 TargetIDSetting SramEcc =
1363 getTargetIDSettingFromModuleFlag(M, FlagName: "amdgpu.sramecc");
1364
1365 SmallString<128> SubtargetKey(GPU);
1366 SubtargetKey.append(RHS: FS);
1367 if (BufRelaxed)
1368 SubtargetKey.append(RHS: ",buf-oob=1");
1369 if (TBufRelaxed)
1370 SubtargetKey.append(RHS: ",tbuf-oob=1");
1371 if (Xnack != TargetIDSetting::Any) {
1372 SubtargetKey.append(RHS: ",xnack=");
1373 SubtargetKey.push_back(Elt: Xnack == TargetIDSetting::On ? '1' : '0');
1374 }
1375 if (SramEcc != TargetIDSetting::Any) {
1376 SubtargetKey.append(RHS: ",sramecc=");
1377 SubtargetKey.push_back(Elt: Xnack == TargetIDSetting::On ? '1' : '0');
1378 }
1379
1380 auto &I = SubtargetMap[SubtargetKey];
1381 if (!I) {
1382 AMDGPU::GPUKind Kind = AMDGPU::parseArchAMDGCN(CPU: GPU);
1383 Triple::SubArchType GPUSubArch = AMDGPU::getSubArch(AK: Kind);
1384
1385 // Enforce the subtarget is covered by the subarch. Tolerate no subarch for
1386 // legacy compatibility.
1387 const Triple &TT = M.getTargetTriple();
1388 if (GPUSubArch != TT.getSubArch() && Kind != AMDGPU::GK_NONE) {
1389 // Check if this is a generic subarch which has subtargets. Ignore
1390 // unknown subtargets with a known subarch, since for whatever reason
1391 // the convention is to just print a warning and ignore unrecognized
1392 // subtargets.
1393 bool IsLegacyEmptySubArch = TT.getSubArch() == Triple::NoSubArch;
1394 if (!IsLegacyEmptySubArch &&
1395 AMDGPU::getMajorSubArch(SubArch: GPUSubArch) != TT.getSubArch()) {
1396 F.getContext().emitError(ErrorStr: "invalid subtarget '" + Twine(GPU) +
1397 "' for subarch " + TT.getArchName());
1398 }
1399 }
1400
1401 I = std::make_unique<GCNSubtarget>(args: TargetTriple, args&: GPU, args&: FS, args: *this, args&: BufRelaxed,
1402 args&: TBufRelaxed, args&: Xnack, args&: SramEcc);
1403 }
1404
1405 return I.get();
1406}
1407
1408TargetTransformInfo
1409GCNTargetMachine::getTargetTransformInfo(const Function &F) const {
1410 return TargetTransformInfo(std::make_unique<GCNTTIImpl>(args: this, args: F));
1411}
1412
1413Error GCNTargetMachine::buildCodeGenPipeline(
1414 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
1415 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
1416 const CGPassBuilderOption &Opts, MCContext &Ctx,
1417 PassInstrumentationCallbacks *PIC) {
1418 AMDGPUCodeGenPassBuilder CGPB(*this, Opts, PIC);
1419 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
1420}
1421
1422ScheduleDAGInstrs *
1423GCNTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
1424 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
1425 if (ST.enableSIScheduler())
1426 return createSIMachineScheduler(C);
1427
1428 StringRef SchedStrategy = AMDGPU::getSchedStrategy(F: C->MF->getFunction());
1429
1430 if (SchedStrategy == "max-ilp")
1431 return createGCNMaxILPMachineScheduler(C);
1432
1433 if (SchedStrategy == "max-memory-clause")
1434 return createGCNMaxMemoryClauseMachineScheduler(C);
1435
1436 if (SchedStrategy == "iterative-ilp")
1437 return createIterativeILPMachineScheduler(C);
1438
1439 if (SchedStrategy == "iterative-minreg")
1440 return createMinRegScheduler(C);
1441
1442 if (SchedStrategy == "iterative-maxocc")
1443 return createIterativeGCNMaxOccupancyMachineScheduler(C);
1444
1445 if (SchedStrategy == "coexec") {
1446 diagnoseUnsupportedCoExecSchedulerSelection(F: C->MF->getFunction(), ST);
1447 return createGCNCoExecMachineScheduler(C);
1448 }
1449
1450 return createGCNMaxOccupancyMachineScheduler(C);
1451}
1452
1453ScheduleDAGInstrs *
1454GCNTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
1455 if (useNoopPostScheduler(F: C->MF->getFunction()))
1456 return createGCNNoopPostMachineScheduler(C);
1457
1458 ScheduleDAGMI *DAG =
1459 new GCNPostScheduleDAGMILive(C, std::make_unique<PostGenericScheduler>(args&: C),
1460 /*RemoveKillFlags=*/true);
1461 const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
1462 DAG->addMutation(Mutation: createLoadClusterDAGMutation(TII: DAG->TII));
1463 if (ST.shouldClusterStores())
1464 DAG->addMutation(Mutation: createStoreClusterDAGMutation(TII: DAG->TII));
1465 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::PostRA));
1466 if ((EnableVOPD.getNumOccurrences() ||
1467 getOptLevel() >= CodeGenOptLevel::Less) &&
1468 EnableVOPD)
1469 DAG->addMutation(Mutation: createVOPDPairingMutation());
1470 DAG->addMutation(Mutation: createAMDGPUExportClusteringDAGMutation());
1471 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
1472 DAG->addMutation(Mutation: createAMDGPUHazardLatencyDAGMutation(MF: C->MF));
1473 return DAG;
1474}
1475//===----------------------------------------------------------------------===//
1476// AMDGPU Legacy Pass Setup
1477//===----------------------------------------------------------------------===//
1478
1479std::unique_ptr<CSEConfigBase> llvm::AMDGPUPassConfig::getCSEConfig() const {
1480 return getStandardCSEConfigForOpt(Level: TM->getOptLevel());
1481}
1482
1483namespace {
1484
1485class GCNPassConfig final : public AMDGPUPassConfig {
1486public:
1487 GCNPassConfig(TargetMachine &TM, PassManagerBase &PM)
1488 : AMDGPUPassConfig(TM, PM) {
1489 substitutePass(StandardID: &PostRASchedulerID, TargetID: &PostMachineSchedulerID);
1490 }
1491
1492 bool addPreISel() override;
1493 void addMachineSSAOptimization() override;
1494 bool addILPOpts() override;
1495 bool addInstSelector() override;
1496 bool addIRTranslator() override;
1497 void addPreLegalizeMachineIR() override;
1498 bool addLegalizeMachineIR() override;
1499 void addPreRegBankSelect() override;
1500 bool addRegBankSelect() override;
1501 void addPreGlobalInstructionSelect() override;
1502 bool addGlobalInstructionSelect() override;
1503 void addPreRegAlloc() override;
1504 void addFastRegAlloc() override;
1505 void addOptimizedRegAlloc() override;
1506
1507 FunctionPass *createSGPRAllocPass(bool Optimized);
1508 FunctionPass *createVGPRAllocPass(bool Optimized);
1509 FunctionPass *createWWMRegAllocPass(bool Optimized);
1510 FunctionPass *createRegAllocPass(bool Optimized) override;
1511
1512 bool addRegAssignAndRewriteFast() override;
1513 bool addRegAssignAndRewriteOptimized() override;
1514
1515 bool addPreRewrite() override;
1516 void addPostRegAlloc() override;
1517 void addPreSched2() override;
1518 void addPreEmitPass() override;
1519 void addPostBBSections() override;
1520};
1521
1522} // end anonymous namespace
1523
1524AMDGPUPassConfig::AMDGPUPassConfig(TargetMachine &TM, PassManagerBase &PM)
1525 : TargetPassConfig(TM, PM) {
1526 // Exceptions and StackMaps are not supported, so these passes will never do
1527 // anything.
1528 disablePass(PassID: &StackMapLivenessID);
1529 disablePass(PassID: &FuncletLayoutID);
1530 // Garbage collection is not supported.
1531 disablePass(PassID: &GCLoweringID);
1532 disablePass(PassID: &ShadowStackGCLoweringID);
1533
1534 if (UseSSAMachineScheduler) {
1535 // Use SSA Machine Scheduler instead of regular Machine Scheduler.
1536 disablePass(PassID: &MachineSchedulerID);
1537 setEnableSSAMachineScheduler(true);
1538 }
1539}
1540
1541void AMDGPUPassConfig::addEarlyCSEOrGVNPass() {
1542 if (getOptLevel() == CodeGenOptLevel::Aggressive)
1543 addPass(P: createGVNPass());
1544 else
1545 addPass(P: createEarlyCSEPass());
1546}
1547
1548void AMDGPUPassConfig::addStraightLineScalarOptimizationPasses() {
1549 if (isPassEnabled(Opt: EnableLoopPrefetch, Level: CodeGenOptLevel::Aggressive))
1550 addPass(P: createLoopDataPrefetchPass());
1551 addPass(P: createSeparateConstOffsetFromGEPPass());
1552 // ReassociateGEPs exposes more opportunities for SLSR. See
1553 // the example in reassociate-geps-and-slsr.ll.
1554 addPass(P: createStraightLineStrengthReducePass());
1555 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or
1556 // EarlyCSE can reuse.
1557 addEarlyCSEOrGVNPass();
1558 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
1559 addPass(P: createNaryReassociatePass());
1560 // NaryReassociate on GEPs creates redundant common expressions, so run
1561 // EarlyCSE after it.
1562 addPass(P: createEarlyCSEPass());
1563}
1564
1565void AMDGPUPassConfig::addIRPasses() {
1566 const AMDGPUTargetMachine &TM = getAMDGPUTargetMachine();
1567
1568 if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN())
1569 addPass(P: createAMDGPURemoveIncompatibleFunctionsPass(&TM));
1570
1571 // There is no reason to run these.
1572 disablePass(PassID: &StackMapLivenessID);
1573 disablePass(PassID: &FuncletLayoutID);
1574 disablePass(PassID: &PatchableFunctionID);
1575
1576 if (TM.getTargetTriple().isAMDGCN())
1577 addPass(P: createAMDGPUPrintfRuntimeBinding());
1578
1579 if (LowerCtorDtor)
1580 addPass(P: createAMDGPUCtorDtorLoweringLegacyPass());
1581
1582 if (TM.getTargetTriple().isAMDGCN() &&
1583 isPassEnabled(Opt: EnableImageIntrinsicOptimizer))
1584 addPass(P: createAMDGPUImageIntrinsicOptimizerPass(&TM));
1585
1586 if (EnableUniformIntrinsicCombine)
1587 addPass(P: createAMDGPUUniformIntrinsicCombineLegacyPass());
1588
1589 // This can be disabled by passing ::Disable here or on the command line
1590 // with --expand-variadics-override=disable.
1591 addPass(P: createExpandVariadicsPass(ExpandVariadicsMode::Lowering));
1592
1593 // Function calls are not supported, so make sure we inline everything.
1594 addPass(P: createAMDGPUAlwaysInlinePass());
1595 addPass(P: createAlwaysInlinerLegacyPass());
1596
1597 // Handle uses of OpenCL image2d_t, image3d_t and sampler_t arguments.
1598 if (TM.getTargetTriple().getArch() == Triple::r600)
1599 addPass(P: createR600OpenCLImageTypeLoweringPass());
1600
1601 // Make enqueued block runtime handles externally visible.
1602 addPass(P: createAMDGPUExportKernelRuntimeHandlesLegacyPass());
1603
1604 // Lower special LDS accesses.
1605 if (EnableLowerExecSync)
1606 addPass(P: createAMDGPULowerExecSyncLegacyPass());
1607
1608 // Lower LDS accesses to global memory pass if address sanitizer is enabled.
1609 if (EnableSwLowerLDS)
1610 addPass(P: createAMDGPUSwLowerLDSLegacyPass());
1611
1612 // Runs before PromoteAlloca so the latter can account for function uses
1613 if (EnableLowerModuleLDS) {
1614 addPass(P: createAMDGPULowerModuleLDSLegacyPass(TM: &TM));
1615 }
1616
1617 // Run atomic optimizer before Atomic Expand
1618 if ((TM.getTargetTriple().isAMDGCN()) &&
1619 (TM.getOptLevel() >= CodeGenOptLevel::Less) &&
1620 (AMDGPUAtomicOptimizerStrategy != ScanOptions::None)) {
1621 addPass(P: createAMDGPUAtomicOptimizerPass(ScanStrategy: AMDGPUAtomicOptimizerStrategy));
1622 }
1623
1624 addPass(P: createAtomicExpandLegacyPass());
1625
1626 if (TM.getOptLevel() > CodeGenOptLevel::None) {
1627 addPass(P: createAMDGPUPromoteAlloca());
1628
1629 if (isPassEnabled(Opt: EnableScalarIRPasses))
1630 addStraightLineScalarOptimizationPasses();
1631
1632 if (EnableAMDGPUAliasAnalysis) {
1633 addPass(P: createAMDGPUAAWrapperPass());
1634 addPass(P: createExternalAAWrapperPass(Callback: [](Pass &P, Function &,
1635 AAResults &AAR) {
1636 if (auto *WrapperPass = P.getAnalysisIfAvailable<AMDGPUAAWrapperPass>())
1637 AAR.addAAResult(AAResult&: WrapperPass->getResult());
1638 }));
1639 }
1640
1641 if (TM.getTargetTriple().isAMDGCN()) {
1642 // TODO: May want to move later or split into an early and late one.
1643 addPass(P: createAMDGPUCodeGenPreparePass());
1644 }
1645
1646 // Try to hoist loop invariant parts of divisions AMDGPUCodeGenPrepare may
1647 // have expanded.
1648 if (TM.getOptLevel() > CodeGenOptLevel::Less)
1649 addPass(P: createLICMPass());
1650 }
1651
1652 TargetPassConfig::addIRPasses();
1653
1654 // EarlyCSE is not always strong enough to clean up what LSR produces. For
1655 // example, GVN can combine
1656 //
1657 // %0 = add %a, %b
1658 // %1 = add %b, %a
1659 //
1660 // and
1661 //
1662 // %0 = shl nsw %a, 2
1663 // %1 = shl %a, 2
1664 //
1665 // but EarlyCSE can do neither of them.
1666 if (isPassEnabled(Opt: EnableScalarIRPasses))
1667 addEarlyCSEOrGVNPass();
1668}
1669
1670void AMDGPUPassConfig::addCodeGenPrepare() {
1671 if (TM->getTargetTriple().isAMDGCN() &&
1672 TM->getOptLevel() > CodeGenOptLevel::None)
1673 addPass(P: createAMDGPUPreloadKernelArgumentsLegacyPass(TM));
1674
1675 if (TM->getTargetTriple().isAMDGCN() && EnableLowerKernelArguments)
1676 addPass(P: createAMDGPULowerKernelArgumentsPass());
1677
1678 TargetPassConfig::addCodeGenPrepare();
1679
1680 if (isPassEnabled(Opt: EnableLoadStoreVectorizer))
1681 addPass(P: createLoadStoreVectorizerPass());
1682
1683 if (TM->getTargetTriple().isAMDGCN()) {
1684 // This lowering has been placed after codegenprepare to take advantage of
1685 // address mode matching (which is why it isn't put with the LDS lowerings).
1686 // It could be placed anywhere before uniformity annotations (an analysis
1687 // that it changes by splitting up fat pointers into their components)
1688 // but has been put before switch lowering and CFG flattening so that those
1689 // passes can run on the more optimized control flow this pass creates in
1690 // many cases.
1691 addPass(P: createAMDGPULowerBufferFatPointersPass());
1692 addPass(P: createAMDGPULowerIntrinsicsLegacyPass());
1693 }
1694
1695 // LowerSwitch pass may introduce unreachable blocks that can
1696 // cause unexpected behavior for subsequent passes. Placing it
1697 // here seems better that these blocks would get cleaned up by
1698 // UnreachableBlockElim inserted next in the pass flow.
1699 addPass(P: createLowerSwitchPass());
1700}
1701
1702bool AMDGPUPassConfig::addPreISel() {
1703 if (TM->getOptLevel() > CodeGenOptLevel::None)
1704 addPass(P: createFlattenCFGPass());
1705 return false;
1706}
1707
1708bool AMDGPUPassConfig::addInstSelector() {
1709 addPass(P: createAMDGPUISelDag(TM&: getAMDGPUTargetMachine(), OptLevel: getOptLevel()));
1710 return false;
1711}
1712
1713bool AMDGPUPassConfig::addGCPasses() {
1714 // Do nothing. GC is not supported.
1715 return false;
1716}
1717
1718//===----------------------------------------------------------------------===//
1719// GCN Legacy Pass Setup
1720//===----------------------------------------------------------------------===//
1721
1722bool GCNPassConfig::addPreISel() {
1723 AMDGPUPassConfig::addPreISel();
1724
1725 if (TM->getOptLevel() > CodeGenOptLevel::None) {
1726 addPass(P: createSinkingPass());
1727 addPass(P: createAMDGPULateCodeGenPrepareLegacyPass());
1728 }
1729
1730 // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
1731 // regions formed by them.
1732 addPass(PassID: &AMDGPUUnifyDivergentExitNodesID);
1733 addPass(P: createFixIrreduciblePass());
1734 addPass(P: createUnifyLoopExitsPass());
1735 addPass(P: createStructurizeCFGPass(/*SkipUniformRegions=*/true));
1736
1737 addPass(P: createAMDGPUAnnotateUniformValuesLegacy());
1738 addPass(P: createSIAnnotateControlFlowLegacyPass());
1739 // TODO: Move this right after structurizeCFG to avoid extra divergence
1740 // analysis. This depends on stopping SIAnnotateControlFlow from making
1741 // control flow modifications.
1742 addPass(P: createAMDGPURewriteUndefForPHILegacyPass());
1743
1744 // SDAG requires LCSSA, GlobalISel does not. Disable LCSSA for -global-isel
1745 // without any of the fallback options.
1746 if (getCGPassBuilderOption().EnableGlobalISelOption !=
1747 cl::boolOrDefault::BOU_TRUE ||
1748 !isGlobalISelAbortEnabled())
1749 addPass(P: createLCSSAPass());
1750
1751 if (TM->getOptLevel() > CodeGenOptLevel::Less)
1752 addPass(PassID: &AMDGPUPerfHintAnalysisLegacyID);
1753
1754 return false;
1755}
1756
1757void GCNPassConfig::addMachineSSAOptimization() {
1758 TargetPassConfig::addMachineSSAOptimization();
1759
1760 // We want to fold operands after PeepholeOptimizer has run (or as part of
1761 // it), because it will eliminate extra copies making it easier to fold the
1762 // real source operand. We want to eliminate dead instructions after, so that
1763 // we see fewer uses of the copies. We then need to clean up the dead
1764 // instructions leftover after the operands are folded as well.
1765 //
1766 // XXX - Can we get away without running DeadMachineInstructionElim again?
1767 addPass(PassID: &SIFoldOperandsLegacyID);
1768 if (EnableDPPCombine)
1769 addPass(PassID: &GCNDPPCombineLegacyID);
1770 addPass(PassID: &SILoadStoreOptimizerLegacyID);
1771 if (isPassEnabled(Opt: EnableSDWAPeephole)) {
1772 addPass(PassID: &SIPeepholeSDWALegacyID);
1773 addPass(PassID: &EarlyMachineLICMID);
1774 addPass(PassID: &MachineCSELegacyID);
1775 addPass(PassID: &SIFoldOperandsLegacyID);
1776 }
1777 addPass(PassID: &DeadMachineInstructionElimID);
1778 addPass(P: createSIShrinkInstructionsLegacyPass());
1779}
1780
1781bool GCNPassConfig::addILPOpts() {
1782 if (EnableEarlyIfConversion)
1783 addPass(PassID: &EarlyIfConverterLegacyID);
1784
1785 TargetPassConfig::addILPOpts();
1786 return false;
1787}
1788
1789bool GCNPassConfig::addInstSelector() {
1790 AMDGPUPassConfig::addInstSelector();
1791 addPass(PassID: &SIFixSGPRCopiesLegacyID);
1792 addPass(P: createSILowerI1CopiesLegacyPass());
1793 return false;
1794}
1795
1796bool GCNPassConfig::addIRTranslator() {
1797 addPass(P: new IRTranslatorLegacy(getOptLevel()));
1798 return false;
1799}
1800
1801void GCNPassConfig::addPreLegalizeMachineIR() {
1802 bool IsOptLevelNone = getOptLevel() == CodeGenOptLevel::None;
1803 addPass(P: createAMDGPUPreLegalizeCombinerLegacyPass(IsOptLevelNone));
1804 addPass(P: new LocalizerLegacy());
1805}
1806
1807bool GCNPassConfig::addLegalizeMachineIR() {
1808 addPass(P: new LegalizerLegacy());
1809 return false;
1810}
1811
1812void GCNPassConfig::addPreRegBankSelect() {
1813 bool IsOptNone = getOptLevel() == CodeGenOptLevel::None;
1814 addPass(P: createAMDGPUPostLegalizeCombinerLegacy(IsOptNone));
1815 addPass(P: createAMDGPUGlobalISelDivergenceLoweringPass());
1816}
1817
1818bool GCNPassConfig::addRegBankSelect() {
1819 addPass(P: createAMDGPURegBankSelectLegacyPass());
1820 addPass(P: createAMDGPURegBankLegalizeLegacyPass());
1821 return false;
1822}
1823
1824void GCNPassConfig::addPreGlobalInstructionSelect() {
1825 bool IsOptLevelNone = getOptLevel() == CodeGenOptLevel::None;
1826 addPass(P: createAMDGPURegBankCombinerLegacy(IsOptLevelNone));
1827}
1828
1829bool GCNPassConfig::addGlobalInstructionSelect() {
1830 addPass(P: new InstructionSelectLegacy(getOptLevel()));
1831 return false;
1832}
1833
1834void GCNPassConfig::addFastRegAlloc() {
1835 // FIXME: We have to disable the verifier here because of PHIElimination +
1836 // TwoAddressInstructions disabling it.
1837
1838 // This must be run immediately after phi elimination and before
1839 // TwoAddressInstructions, otherwise the processing of the tied operand of
1840 // SI_ELSE will introduce a copy of the tied operand source after the else.
1841 insertPass(TargetPassID: &PHIEliminationID, InsertedPassID: &SILowerControlFlowLegacyID);
1842
1843 insertPass(TargetPassID: &TwoAddressInstructionPassID, InsertedPassID: &SIWholeQuadModeID);
1844
1845 TargetPassConfig::addFastRegAlloc();
1846}
1847
1848void GCNPassConfig::addPreRegAlloc() {
1849 if (getOptLevel() != CodeGenOptLevel::None)
1850 addPass(PassID: &AMDGPUPrepareAGPRAllocLegacyID);
1851 if (getOptLevel() >= CodeGenOptLevel::Default && EnableMachinePipeliner)
1852 addPass(PassID: &MachinePipelinerID);
1853}
1854
1855void GCNPassConfig::addOptimizedRegAlloc() {
1856 if (EnableDCEInRA)
1857 insertPass(TargetPassID: &DetectDeadLanesID, InsertedPassID: &DeadMachineInstructionElimID);
1858
1859 if (OptVGPRLiveRange)
1860 insertPass(TargetPassID: &MachineLoopInfoID, InsertedPassID: &SIOptimizeVGPRLiveRangeLegacyID);
1861
1862 // This must be run immediately after phi elimination and before
1863 // TwoAddressInstructions, otherwise the processing of the tied operand of
1864 // SI_ELSE will introduce a copy of the tied operand source after the else.
1865 insertPass(TargetPassID: &PHIEliminationID, InsertedPassID: &SILowerControlFlowLegacyID);
1866
1867 if (EnableRewritePartialRegUses)
1868 insertPass(TargetPassID: &RenameIndependentSubregsID, InsertedPassID: &GCNRewritePartialRegUsesID);
1869
1870 // Insertion point for passes depends on whether MachineScheduler is enabled.
1871 AnalysisID EndOfPreRA = UseSSAMachineScheduler ? &RenameIndependentSubregsID
1872 : &MachineSchedulerID;
1873
1874 if (isPassEnabled(Opt: EnablePreRAOptimizations))
1875 insertPass(TargetPassID: EndOfPreRA, InsertedPassID: &GCNPreRAOptimizationsID);
1876
1877 // Allow the scheduler to run before SIWholeQuadMode inserts exec manipulation
1878 // instructions that cause scheduling barriers.
1879 insertPass(TargetPassID: EndOfPreRA, InsertedPassID: &SIWholeQuadModeID);
1880
1881 if (OptExecMaskPreRA)
1882 insertPass(TargetPassID: EndOfPreRA, InsertedPassID: &SIOptimizeExecMaskingPreRAID);
1883
1884 // This is not an essential optimization and it has a noticeable impact on
1885 // compilation time, so we only enable it from O2.
1886 if (TM->getOptLevel() > CodeGenOptLevel::Less)
1887 insertPass(TargetPassID: EndOfPreRA, InsertedPassID: &SIFormMemoryClausesID);
1888
1889 TargetPassConfig::addOptimizedRegAlloc();
1890}
1891
1892bool GCNPassConfig::addPreRewrite() {
1893 if (EnableRegReassign)
1894 addPass(PassID: &GCNNSAReassignID);
1895
1896 addPass(PassID: &AMDGPURewriteAGPRCopyMFMALegacyID);
1897 return true;
1898}
1899
1900FunctionPass *GCNPassConfig::createSGPRAllocPass(bool Optimized) {
1901 // Initialize the global default.
1902 llvm::call_once(flag&: InitializeDefaultSGPRRegisterAllocatorFlag,
1903 F&: initializeDefaultSGPRRegisterAllocatorOnce);
1904
1905 RegisterRegAlloc::FunctionPassCtor Ctor = SGPRRegisterRegAlloc::getDefault();
1906 if (Ctor != useDefaultRegisterAllocator)
1907 return Ctor();
1908
1909 if (Optimized)
1910 return createGreedyRegisterAllocator(F: onlyAllocateSGPRs);
1911
1912 return createFastRegisterAllocator(F: onlyAllocateSGPRs, ClearVirtRegs: false);
1913}
1914
1915FunctionPass *GCNPassConfig::createVGPRAllocPass(bool Optimized) {
1916 // Initialize the global default.
1917 llvm::call_once(flag&: InitializeDefaultVGPRRegisterAllocatorFlag,
1918 F&: initializeDefaultVGPRRegisterAllocatorOnce);
1919
1920 RegisterRegAlloc::FunctionPassCtor Ctor = VGPRRegisterRegAlloc::getDefault();
1921 if (Ctor != useDefaultRegisterAllocator)
1922 return Ctor();
1923
1924 if (Optimized)
1925 return createGreedyVGPRRegisterAllocator();
1926
1927 return createFastVGPRRegisterAllocator();
1928}
1929
1930FunctionPass *GCNPassConfig::createWWMRegAllocPass(bool Optimized) {
1931 // Initialize the global default.
1932 llvm::call_once(flag&: InitializeDefaultWWMRegisterAllocatorFlag,
1933 F&: initializeDefaultWWMRegisterAllocatorOnce);
1934
1935 RegisterRegAlloc::FunctionPassCtor Ctor = WWMRegisterRegAlloc::getDefault();
1936 if (Ctor != useDefaultRegisterAllocator)
1937 return Ctor();
1938
1939 if (Optimized)
1940 return createGreedyWWMRegisterAllocator();
1941
1942 return createFastWWMRegisterAllocator();
1943}
1944
1945FunctionPass *GCNPassConfig::createRegAllocPass(bool Optimized) {
1946 llvm_unreachable("should not be used");
1947}
1948
1949static const char RegAllocOptNotSupportedMessage[] =
1950 "-regalloc not supported with amdgcn. Use -sgpr-regalloc, -wwm-regalloc, "
1951 "and -vgpr-regalloc";
1952
1953bool GCNPassConfig::addRegAssignAndRewriteFast() {
1954 if (!usingDefaultRegAlloc())
1955 reportFatalUsageError(reason: RegAllocOptNotSupportedMessage);
1956
1957 addPass(PassID: &GCNPreRALongBranchRegID);
1958
1959 addPass(P: createSGPRAllocPass(Optimized: false));
1960
1961 // Equivalent of PEI for SGPRs.
1962 addPass(PassID: &SILowerSGPRSpillsLegacyID);
1963
1964 // To Allocate wwm registers used in whole quad mode operations (for shaders).
1965 addPass(PassID: &SIPreAllocateWWMRegsLegacyID);
1966
1967 // For allocating other wwm register operands.
1968 addPass(P: createWWMRegAllocPass(Optimized: false));
1969
1970 addPass(PassID: &SILowerWWMCopiesLegacyID);
1971 addPass(PassID: &AMDGPUReserveWWMRegsLegacyID);
1972
1973 // For allocating per-thread VGPRs.
1974 addPass(P: createVGPRAllocPass(Optimized: false));
1975
1976 return true;
1977}
1978
1979bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
1980 if (!usingDefaultRegAlloc())
1981 reportFatalUsageError(reason: RegAllocOptNotSupportedMessage);
1982
1983 addPass(PassID: &GCNPreRALongBranchRegID);
1984
1985 addPass(P: createSGPRAllocPass(Optimized: true));
1986
1987 // Commit allocated register changes. This is mostly necessary because too
1988 // many things rely on the use lists of the physical registers, such as the
1989 // verifier. This is only necessary with allocators which use LiveIntervals,
1990 // since FastRegAlloc does the replacements itself.
1991 addPass(P: createVirtRegRewriter(ClearVirtRegs: false));
1992
1993 // At this point, the sgpr-regalloc has been done and it is good to have the
1994 // stack slot coloring to try to optimize the SGPR spill stack indices before
1995 // attempting the custom SGPR spill lowering.
1996 addPass(PassID: &StackSlotColoringID);
1997
1998 // Equivalent of PEI for SGPRs.
1999 addPass(PassID: &SILowerSGPRSpillsLegacyID);
2000
2001 // To Allocate wwm registers used in whole quad mode operations (for shaders).
2002 addPass(PassID: &SIPreAllocateWWMRegsLegacyID);
2003
2004 // For allocating other whole wave mode registers.
2005 addPass(P: createWWMRegAllocPass(Optimized: true));
2006 addPass(PassID: &SILowerWWMCopiesLegacyID);
2007 addPass(P: createVirtRegRewriter(ClearVirtRegs: false));
2008 addPass(PassID: &AMDGPUReserveWWMRegsLegacyID);
2009
2010 // For allocating per-thread VGPRs.
2011 addPass(P: createVGPRAllocPass(Optimized: true));
2012
2013 addPreRewrite();
2014 addPass(PassID: &VirtRegRewriterID);
2015
2016 addPass(PassID: &AMDGPUMarkLastScratchLoadID);
2017
2018 return true;
2019}
2020
2021void GCNPassConfig::addPostRegAlloc() {
2022 addPass(PassID: &SIFixVGPRCopiesID);
2023 if (getOptLevel() > CodeGenOptLevel::None)
2024 addPass(PassID: &SIOptimizeExecMaskingLegacyID);
2025 TargetPassConfig::addPostRegAlloc();
2026}
2027
2028void GCNPassConfig::addPreSched2() {
2029 if (TM->getOptLevel() > CodeGenOptLevel::None)
2030 addPass(P: createSIShrinkInstructionsLegacyPass());
2031 addPass(PassID: &SIPostRABundlerLegacyID);
2032}
2033
2034void GCNPassConfig::addPreEmitPass() {
2035 if (isPassEnabled(Opt: EnableVOPD, Level: CodeGenOptLevel::Less))
2036 addPass(PassID: &GCNCreateVOPDID);
2037 addPass(P: createSIMemoryLegalizerPass());
2038 if (getOptLevel() > CodeGenOptLevel::None)
2039 addPass(PassID: &SIPostRA16BitMovFoldingLegacyID);
2040 addPass(P: createSIInsertWaitcntsPass());
2041
2042 addPass(P: createSIModeRegisterPass());
2043
2044 if (getOptLevel() > CodeGenOptLevel::None)
2045 addPass(PassID: &SIInsertHardClausesID);
2046
2047 addPass(PassID: &SILateBranchLoweringPassID);
2048 if (isPassEnabled(Opt: EnableSetWavePriority, Level: CodeGenOptLevel::Less))
2049 addPass(P: createAMDGPUSetWavePriorityPass());
2050 if (getOptLevel() > CodeGenOptLevel::None)
2051 addPass(PassID: &SIPreEmitPeepholeID);
2052 // The hazard recognizer that runs as part of the post-ra scheduler does not
2053 // guarantee to be able handle all hazards correctly. This is because if there
2054 // are multiple scheduling regions in a basic block, the regions are scheduled
2055 // bottom up, so when we begin to schedule a region we don't know what
2056 // instructions were emitted directly before it.
2057 //
2058 // Here we add a stand-alone hazard recognizer pass which can handle all
2059 // cases.
2060 addPass(PassID: &PostRAHazardRecognizerID);
2061
2062 addPass(PassID: &AMDGPUWaitSGPRHazardsLegacyID);
2063
2064 addPass(PassID: &AMDGPULowerVGPREncodingLegacyID);
2065
2066 if (isPassEnabled(Opt: EnableInsertDelayAlu, Level: CodeGenOptLevel::Less))
2067 addPass(PassID: &AMDGPUInsertDelayAluID);
2068
2069 addPass(PassID: &BranchRelaxationPassID);
2070}
2071
2072void GCNPassConfig::addPostBBSections() {
2073 // We run this later to avoid passes like livedebugvalues and BBSections
2074 // having to deal with the apparent multi-entry functions we may generate.
2075 addPass(P: createAMDGPUPreloadKernArgPrologLegacyPass());
2076}
2077
2078TargetPassConfig *GCNTargetMachine::createPassConfig(PassManagerBase &PM) {
2079 return new GCNPassConfig(*this, PM);
2080}
2081
2082void GCNTargetMachine::registerMachineRegisterInfoCallback(
2083 MachineFunction &MF) const {
2084 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2085 MF.getRegInfo().addDelegate(delegate: MFI);
2086}
2087
2088MachineFunctionInfo *GCNTargetMachine::createMachineFunctionInfo(
2089 BumpPtrAllocator &Allocator, const Function &F,
2090 const TargetSubtargetInfo *STI) const {
2091 return SIMachineFunctionInfo::create<SIMachineFunctionInfo>(
2092 Allocator, F, STI: static_cast<const GCNSubtarget *>(STI));
2093}
2094
2095yaml::MachineFunctionInfo *GCNTargetMachine::createDefaultFuncInfoYAML() const {
2096 return new yaml::SIMachineFunctionInfo();
2097}
2098
2099yaml::MachineFunctionInfo *
2100GCNTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
2101 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2102 return new yaml::SIMachineFunctionInfo(
2103 *MFI, *MF.getSubtarget<GCNSubtarget>().getRegisterInfo(), MF);
2104}
2105
2106bool GCNTargetMachine::parseMachineFunctionInfo(
2107 const yaml::MachineFunctionInfo &MFI_, PerFunctionMIParsingState &PFS,
2108 SMDiagnostic &Error, SMRange &SourceRange) const {
2109 const yaml::SIMachineFunctionInfo &YamlMFI =
2110 static_cast<const yaml::SIMachineFunctionInfo &>(MFI_);
2111 MachineFunction &MF = PFS.MF;
2112 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2113 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2114
2115 if (MFI->initializeBaseYamlFields(YamlMFI, MF, PFS, Error, SourceRange))
2116 return true;
2117
2118 if (MFI->Occupancy == 0) {
2119 // Fixup the subtarget dependent default value.
2120 MFI->Occupancy = ST.getOccupancyWithWorkGroupSizes(MF).second;
2121 }
2122
2123 auto parseRegister = [&](const yaml::StringValue &RegName, Register &RegVal) {
2124 Register TempReg;
2125 if (parseNamedRegisterReference(PFS, Reg&: TempReg, Src: RegName.Value, Error)) {
2126 SourceRange = RegName.SourceRange;
2127 return true;
2128 }
2129 RegVal = TempReg;
2130
2131 return false;
2132 };
2133
2134 auto parseOptionalRegister = [&](const yaml::StringValue &RegName,
2135 Register &RegVal) {
2136 return !RegName.Value.empty() && parseRegister(RegName, RegVal);
2137 };
2138
2139 if (parseOptionalRegister(YamlMFI.VGPRForAGPRCopy, MFI->VGPRForAGPRCopy))
2140 return true;
2141
2142 if (parseOptionalRegister(YamlMFI.SGPRForEXECCopy, MFI->SGPRForEXECCopy))
2143 return true;
2144
2145 if (parseOptionalRegister(YamlMFI.LongBranchReservedReg,
2146 MFI->LongBranchReservedReg))
2147 return true;
2148
2149 auto diagnoseRegisterClass = [&](const yaml::StringValue &RegName) {
2150 // Create a diagnostic for a the register string literal.
2151 const MemoryBuffer &Buffer =
2152 *PFS.SM->getMemoryBuffer(i: PFS.SM->getMainFileID());
2153 Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1,
2154 RegName.Value.size(), SourceMgr::DK_Error,
2155 "incorrect register class for field", RegName.Value,
2156 {}, {});
2157 SourceRange = RegName.SourceRange;
2158 return true;
2159 };
2160
2161 if (parseRegister(YamlMFI.ScratchRSrcReg, MFI->ScratchRSrcReg) ||
2162 parseRegister(YamlMFI.FrameOffsetReg, MFI->FrameOffsetReg) ||
2163 parseRegister(YamlMFI.StackPtrOffsetReg, MFI->StackPtrOffsetReg))
2164 return true;
2165
2166 if (MFI->ScratchRSrcReg != AMDGPU::PRIVATE_RSRC_REG &&
2167 !AMDGPU::SGPR_128RegClass.contains(Reg: MFI->ScratchRSrcReg)) {
2168 return diagnoseRegisterClass(YamlMFI.ScratchRSrcReg);
2169 }
2170
2171 if (MFI->FrameOffsetReg != AMDGPU::FP_REG &&
2172 !AMDGPU::SGPR_32RegClass.contains(Reg: MFI->FrameOffsetReg)) {
2173 return diagnoseRegisterClass(YamlMFI.FrameOffsetReg);
2174 }
2175
2176 if (MFI->StackPtrOffsetReg != AMDGPU::SP_REG &&
2177 !AMDGPU::SGPR_32RegClass.contains(Reg: MFI->StackPtrOffsetReg)) {
2178 return diagnoseRegisterClass(YamlMFI.StackPtrOffsetReg);
2179 }
2180
2181 for (const auto &YamlReg : YamlMFI.WWMReservedRegs) {
2182 Register ParsedReg;
2183 if (parseRegister(YamlReg, ParsedReg))
2184 return true;
2185
2186 MFI->reserveWWMRegister(Reg: ParsedReg);
2187 }
2188
2189 for (const auto &[_, Info] : PFS.VRegInfosNamed) {
2190 MFI->setFlag(Reg: Info->VReg, Flag: Info->Flags);
2191 }
2192 for (const auto &[_, Info] : PFS.VRegInfos) {
2193 MFI->setFlag(Reg: Info->VReg, Flag: Info->Flags);
2194 }
2195
2196 for (const auto &YamlRegStr : YamlMFI.SpillPhysVGPRS) {
2197 Register ParsedReg;
2198 if (parseRegister(YamlRegStr, ParsedReg))
2199 return true;
2200 MFI->SpillPhysVGPRs.push_back(Elt: ParsedReg);
2201 }
2202
2203 auto parseAndCheckArgument = [&](const std::optional<yaml::SIArgument> &A,
2204 const TargetRegisterClass &RC,
2205 ArgDescriptor &Arg, unsigned UserSGPRs,
2206 unsigned SystemSGPRs) {
2207 // Skip parsing if it's not present.
2208 if (!A)
2209 return false;
2210
2211 if (A->IsRegister) {
2212 Register Reg;
2213 if (parseNamedRegisterReference(PFS, Reg, Src: A->RegisterName.Value, Error)) {
2214 SourceRange = A->RegisterName.SourceRange;
2215 return true;
2216 }
2217 if (!RC.contains(Reg))
2218 return diagnoseRegisterClass(A->RegisterName);
2219 Arg = ArgDescriptor::createRegister(Reg);
2220 } else
2221 Arg = ArgDescriptor::createStack(Offset: A->StackOffset);
2222 // Check and apply the optional mask.
2223 if (A->Mask)
2224 Arg = ArgDescriptor::createArg(Arg, Mask: *A->Mask);
2225
2226 MFI->NumUserSGPRs += UserSGPRs;
2227 MFI->NumSystemSGPRs += SystemSGPRs;
2228 return false;
2229 };
2230
2231 if (YamlMFI.ArgInfo &&
2232 (parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentBuffer,
2233 AMDGPU::SGPR_128RegClass,
2234 MFI->ArgInfo.PrivateSegmentBuffer, 4, 0) ||
2235 parseAndCheckArgument(YamlMFI.ArgInfo->DispatchPtr,
2236 AMDGPU::SReg_64RegClass, MFI->ArgInfo.DispatchPtr,
2237 2, 0) ||
2238 parseAndCheckArgument(YamlMFI.ArgInfo->QueuePtr, AMDGPU::SReg_64RegClass,
2239 MFI->ArgInfo.QueuePtr, 2, 0) ||
2240 parseAndCheckArgument(YamlMFI.ArgInfo->KernargSegmentPtr,
2241 AMDGPU::SReg_64RegClass,
2242 MFI->ArgInfo.KernargSegmentPtr, 2, 0) ||
2243 parseAndCheckArgument(YamlMFI.ArgInfo->DispatchID,
2244 AMDGPU::SReg_64RegClass, MFI->ArgInfo.DispatchID,
2245 2, 0) ||
2246 parseAndCheckArgument(YamlMFI.ArgInfo->FlatScratchInit,
2247 AMDGPU::SReg_64RegClass,
2248 MFI->ArgInfo.FlatScratchInit, 2, 0) ||
2249 parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentSize,
2250 AMDGPU::SGPR_32RegClass,
2251 MFI->ArgInfo.PrivateSegmentSize, 1, 0) ||
2252 parseAndCheckArgument(YamlMFI.ArgInfo->LDSKernelId,
2253 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.LDSKernelId,
2254 1, 0) ||
2255 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDX,
2256 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDX,
2257 0, 1) ||
2258 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDY,
2259 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDY,
2260 0, 1) ||
2261 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDZ,
2262 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDZ,
2263 0, 1) ||
2264 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupInfo,
2265 AMDGPU::SGPR_32RegClass,
2266 MFI->ArgInfo.WorkGroupInfo, 0, 1) ||
2267 parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentWaveByteOffset,
2268 AMDGPU::SGPR_32RegClass,
2269 MFI->ArgInfo.PrivateSegmentWaveByteOffset, 0, 1) ||
2270 parseAndCheckArgument(YamlMFI.ArgInfo->ImplicitArgPtr,
2271 AMDGPU::SReg_64RegClass,
2272 MFI->ArgInfo.ImplicitArgPtr, 0, 0) ||
2273 parseAndCheckArgument(YamlMFI.ArgInfo->ImplicitBufferPtr,
2274 AMDGPU::SReg_64RegClass,
2275 MFI->ArgInfo.ImplicitBufferPtr, 2, 0) ||
2276 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDX,
2277 AMDGPU::VGPR_32RegClass, MFI->ArgInfo.WorkItemIDX,
2278 0, 0) ||
2279 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDY,
2280 AMDGPU::VGPR_32RegClass, MFI->ArgInfo.WorkItemIDY,
2281 0, 0) ||
2282 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDZ,
2283 AMDGPU::VGPR_32RegClass, MFI->ArgInfo.WorkItemIDZ,
2284 0, 0)))
2285 return true;
2286
2287 // Parse FirstKernArgPreloadReg separately, since it's a Register,
2288 // not ArgDescriptor.
2289 if (YamlMFI.ArgInfo && YamlMFI.ArgInfo->FirstKernArgPreloadReg) {
2290 const yaml::SIArgument &A = *YamlMFI.ArgInfo->FirstKernArgPreloadReg;
2291
2292 if (!A.IsRegister) {
2293 // For stack arguments, we don't have RegisterName.SourceRange,
2294 // but we should have some location info from the YAML parser
2295 const MemoryBuffer &Buffer =
2296 *PFS.SM->getMemoryBuffer(i: PFS.SM->getMainFileID());
2297 // Create a minimal valid source range
2298 SMLoc Loc = SMLoc::getFromPointer(Ptr: Buffer.getBufferStart());
2299 SMRange Range(Loc, Loc);
2300
2301 Error = SMDiagnostic(
2302 *PFS.SM, Loc, Buffer.getBufferIdentifier(), 1, 0, SourceMgr::DK_Error,
2303 "firstKernArgPreloadReg must be a register, not a stack location", "",
2304 {}, {});
2305
2306 SourceRange = Range;
2307 return true;
2308 }
2309
2310 Register Reg;
2311 if (parseNamedRegisterReference(PFS, Reg, Src: A.RegisterName.Value, Error)) {
2312 SourceRange = A.RegisterName.SourceRange;
2313 return true;
2314 }
2315
2316 if (!AMDGPU::SGPR_32RegClass.contains(Reg))
2317 return diagnoseRegisterClass(A.RegisterName);
2318
2319 MFI->ArgInfo.FirstKernArgPreloadReg = Reg;
2320 MFI->NumUserSGPRs += YamlMFI.NumKernargPreloadSGPRs;
2321 }
2322
2323 if (ST.hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode)) {
2324 MFI->Mode.IEEE = YamlMFI.Mode.IEEE;
2325 MFI->Mode.DX10Clamp = YamlMFI.Mode.DX10Clamp;
2326 }
2327
2328 // FIXME: Move proper support for denormal-fp-math into base MachineFunction
2329 MFI->Mode.FP32Denormals.Input = YamlMFI.Mode.FP32InputDenormals
2330 ? DenormalMode::IEEE
2331 : DenormalMode::PreserveSign;
2332 MFI->Mode.FP32Denormals.Output = YamlMFI.Mode.FP32OutputDenormals
2333 ? DenormalMode::IEEE
2334 : DenormalMode::PreserveSign;
2335
2336 MFI->Mode.FP64FP16Denormals.Input = YamlMFI.Mode.FP64FP16InputDenormals
2337 ? DenormalMode::IEEE
2338 : DenormalMode::PreserveSign;
2339 MFI->Mode.FP64FP16Denormals.Output = YamlMFI.Mode.FP64FP16OutputDenormals
2340 ? DenormalMode::IEEE
2341 : DenormalMode::PreserveSign;
2342
2343 if (YamlMFI.HasInitWholeWave)
2344 MFI->setInitWholeWave();
2345
2346 return false;
2347}
2348
2349//===----------------------------------------------------------------------===//
2350// AMDGPU CodeGen Pass Builder interface.
2351//===----------------------------------------------------------------------===//
2352
2353AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
2354 GCNTargetMachine &TM, const CGPassBuilderOption &Opts,
2355 PassInstrumentationCallbacks *PIC)
2356 : CodeGenPassBuilder(TM, Opts, PIC) {
2357 Opt.MISchedPostRA = true;
2358 Opt.RequiresCodeGenSCCOrder = true;
2359 // Exceptions and StackMaps are not supported, so these passes will never do
2360 // anything.
2361 // Garbage collection is not supported.
2362 disablePass<StackMapLivenessPass, FuncletLayoutPass, PatchableFunctionPass,
2363 ShadowStackGCLoweringPass, GCLoweringPass>();
2364}
2365
2366void AMDGPUCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
2367 if (RemoveIncompatibleFunctions && TM.getTargetTriple().isAMDGCN()) {
2368 flushFPMsToMPM(PMW);
2369 addModulePass(Pass: AMDGPURemoveIncompatibleFunctionsPass(TM), PMW);
2370 }
2371
2372 flushFPMsToMPM(PMW);
2373
2374 if (TM.getTargetTriple().isAMDGCN())
2375 addModulePass(Pass: AMDGPUPrintfRuntimeBindingPass(), PMW);
2376
2377 if (LowerCtorDtor)
2378 addModulePass(Pass: AMDGPUCtorDtorLoweringPass(), PMW);
2379
2380 if (isPassEnabled(Opt: EnableImageIntrinsicOptimizer))
2381 addFunctionPass(Pass: AMDGPUImageIntrinsicOptimizerPass(TM), PMW);
2382
2383 if (EnableUniformIntrinsicCombine)
2384 addFunctionPass(Pass: AMDGPUUniformIntrinsicCombinePass(), PMW);
2385 // This can be disabled by passing ::Disable here or on the command line
2386 // with --expand-variadics-override=disable.
2387 flushFPMsToMPM(PMW);
2388 addModulePass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Lowering), PMW);
2389
2390 addModulePass(Pass: AMDGPUAlwaysInlinePass(), PMW);
2391 addModulePass(Pass: AlwaysInlinerPass(), PMW);
2392
2393 addModulePass(Pass: AMDGPUExportKernelRuntimeHandlesPass(), PMW);
2394
2395 if (EnableLowerExecSync)
2396 addModulePass(Pass: AMDGPULowerExecSyncPass(), PMW);
2397
2398 if (EnableSwLowerLDS)
2399 addModulePass(Pass: AMDGPUSwLowerLDSPass(), PMW);
2400
2401 // Runs before PromoteAlloca so the latter can account for function uses
2402 if (EnableLowerModuleLDS)
2403 addModulePass(Pass: AMDGPULowerModuleLDSPass(getTM()), PMW);
2404
2405 // Run atomic optimizer before Atomic Expand
2406 if (TM.getOptLevel() >= CodeGenOptLevel::Less &&
2407 (AMDGPUAtomicOptimizerStrategy != ScanOptions::None))
2408 addFunctionPass(
2409 Pass: AMDGPUAtomicOptimizerPass(TM, AMDGPUAtomicOptimizerStrategy), PMW);
2410
2411 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
2412
2413 if (TM.getOptLevel() > CodeGenOptLevel::None) {
2414 addFunctionPass(Pass: AMDGPUPromoteAllocaPass(TM), PMW);
2415 if (isPassEnabled(Opt: EnableScalarIRPasses))
2416 addStraightLineScalarOptimizationPasses(PMW);
2417
2418 // TODO: Handle EnableAMDGPUAliasAnalysis
2419
2420 // TODO: May want to move later or split into an early and late one.
2421 addFunctionPass(Pass: AMDGPUCodeGenPreparePass(TM), PMW);
2422
2423 // Try to hoist loop invariant parts of divisions AMDGPUCodeGenPrepare may
2424 // have expanded.
2425 if (TM.getOptLevel() > CodeGenOptLevel::Less) {
2426 addFunctionPass(Pass: createFunctionToLoopPassAdaptor(Pass: LICMPass(LICMOptions()),
2427 /*UseMemorySSA=*/true),
2428 PMW);
2429 }
2430 }
2431
2432 Base::addIRPasses(PMW);
2433
2434 // EarlyCSE is not always strong enough to clean up what LSR produces. For
2435 // example, GVN can combine
2436 //
2437 // %0 = add %a, %b
2438 // %1 = add %b, %a
2439 //
2440 // and
2441 //
2442 // %0 = shl nsw %a, 2
2443 // %1 = shl %a, 2
2444 //
2445 // but EarlyCSE can do neither of them.
2446 if (isPassEnabled(Opt: EnableScalarIRPasses))
2447 addEarlyCSEOrGVNPass(PMW);
2448}
2449
2450void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(PassManagerWrapper &PMW) {
2451 if (TM.getOptLevel() > CodeGenOptLevel::None) {
2452 flushFPMsToMPM(PMW);
2453 addModulePass(Pass: AMDGPUPreloadKernelArgumentsPass(TM), PMW);
2454 }
2455
2456 if (EnableLowerKernelArguments)
2457 addFunctionPass(Pass: AMDGPULowerKernelArgumentsPass(TM), PMW);
2458
2459 Base::addCodeGenPrepare(PMW);
2460
2461 if (isPassEnabled(Opt: EnableLoadStoreVectorizer))
2462 addFunctionPass(Pass: LoadStoreVectorizerPass(), PMW);
2463
2464 // This lowering has been placed after codegenprepare to take advantage of
2465 // address mode matching (which is why it isn't put with the LDS lowerings).
2466 // It could be placed anywhere before uniformity annotations (an analysis
2467 // that it changes by splitting up fat pointers into their components)
2468 // but has been put before switch lowering and CFG flattening so that those
2469 // passes can run on the more optimized control flow this pass creates in
2470 // many cases.
2471 flushFPMsToMPM(PMW);
2472 addModulePass(Pass: AMDGPULowerBufferFatPointersPass(TM), PMW);
2473 flushFPMsToMPM(PMW);
2474 requireCGSCCOrder(PMW);
2475
2476 addModulePass(Pass: AMDGPULowerIntrinsicsPass(getTM()), PMW);
2477
2478 // LowerSwitch pass may introduce unreachable blocks that can cause unexpected
2479 // behavior for subsequent passes. Placing it here seems better that these
2480 // blocks would get cleaned up by UnreachableBlockElim inserted next in the
2481 // pass flow.
2482 addFunctionPass(Pass: LowerSwitchPass(), PMW);
2483}
2484
2485void AMDGPUCodeGenPassBuilder::addPreISel(PassManagerWrapper &PMW) {
2486
2487 if (TM.getOptLevel() > CodeGenOptLevel::None) {
2488 addFunctionPass(Pass: FlattenCFGPass(), PMW);
2489 addFunctionPass(Pass: SinkingPass(), PMW);
2490 addFunctionPass(Pass: AMDGPULateCodeGenPreparePass(getTM()), PMW);
2491 }
2492
2493 // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
2494 // regions formed by them.
2495
2496 addFunctionPass(Pass: AMDGPUUnifyDivergentExitNodesPass(), PMW);
2497 addFunctionPass(Pass: FixIrreduciblePass(), PMW);
2498 addFunctionPass(Pass: UnifyLoopExitsPass(), PMW);
2499 addFunctionPass(Pass: StructurizeCFGPass(/*SkipUniformRegions=*/true), PMW);
2500
2501 addFunctionPass(Pass: AMDGPUAnnotateUniformValuesPass(), PMW);
2502
2503 addFunctionPass(Pass: SIAnnotateControlFlowPass(getTM()), PMW);
2504
2505 // TODO: Move this right after structurizeCFG to avoid extra divergence
2506 // analysis. This depends on stopping SIAnnotateControlFlow from making
2507 // control flow modifications.
2508 addFunctionPass(Pass: AMDGPURewriteUndefForPHIPass(), PMW);
2509
2510 if (getCGPassBuilderOption().EnableGlobalISelOption !=
2511 cl::boolOrDefault::BOU_TRUE ||
2512 !isGlobalISelAbortEnabled())
2513 addFunctionPass(Pass: LCSSAPass(), PMW);
2514
2515 if (TM.getOptLevel() > CodeGenOptLevel::Less) {
2516 flushFPMsToMPM(PMW);
2517 addModulePass(Pass: AMDGPUPerfHintAnalysisPass(getTM()), PMW);
2518 }
2519}
2520
2521void AMDGPUCodeGenPassBuilder::addILPOpts(PassManagerWrapper &PMW) {
2522 if (EnableEarlyIfConversion)
2523 addMachineFunctionPass(Pass: EarlyIfConverterPass(), PMW);
2524
2525 Base::addILPOpts(PMW);
2526}
2527
2528void AMDGPUCodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
2529 addModulePass(Pass: AMDGPUAsmPrinterBeginPass(), PMW,
2530 /*Force=*/true);
2531}
2532
2533void AMDGPUCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
2534 addMachineFunctionPass(Pass: AMDGPUAsmPrinterPass(), PMW);
2535}
2536
2537void AMDGPUCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
2538 addModulePass(Pass: AMDGPUAsmPrinterEndPass(), PMW);
2539}
2540
2541Error AMDGPUCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
2542 addMachineFunctionPass(Pass: AMDGPUISelDAGToDAGPass(TM), PMW);
2543 addMachineFunctionPass(Pass: SIFixSGPRCopiesPass(), PMW);
2544 addMachineFunctionPass(Pass: SILowerI1CopiesPass(), PMW);
2545 return Error::success();
2546}
2547
2548Error AMDGPUCodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
2549 addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW);
2550 return Error::success();
2551}
2552
2553void AMDGPUCodeGenPassBuilder::addPreLegalizeMachineIR(
2554 PassManagerWrapper &PMW) {
2555 addMachineFunctionPass(Pass: AMDGPUPreLegalizerCombinerPass(), PMW);
2556 addMachineFunctionPass(Pass: LocalizerPass(), PMW);
2557}
2558
2559Error AMDGPUCodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
2560 addMachineFunctionPass(Pass: LegalizerPass(), PMW);
2561 return Error::success();
2562}
2563
2564void AMDGPUCodeGenPassBuilder::addPreRegBankSelect(PassManagerWrapper &PMW) {
2565 addMachineFunctionPass(Pass: AMDGPUPostLegalizerCombinerPass(), PMW);
2566 addMachineFunctionPass(Pass: AMDGPUGlobalISelDivergenceLoweringPass(), PMW);
2567}
2568
2569Error AMDGPUCodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
2570 addMachineFunctionPass(Pass: AMDGPURegBankSelectPass(), PMW);
2571 addMachineFunctionPass(Pass: AMDGPURegBankLegalizePass(), PMW);
2572 return Error::success();
2573}
2574
2575void AMDGPUCodeGenPassBuilder::addPreGlobalInstructionSelect(
2576 PassManagerWrapper &PMW) {
2577 bool IsOptLevelNone = getOptLevel() == CodeGenOptLevel::None;
2578 addMachineFunctionPass(Pass: AMDGPURegBankCombinerPass(IsOptLevelNone), PMW);
2579}
2580
2581Error AMDGPUCodeGenPassBuilder::addGlobalInstructionSelect(
2582 PassManagerWrapper &PMW) {
2583 addMachineFunctionPass(Pass: InstructionSelectPass(getOptLevel()), PMW);
2584 return Error::success();
2585}
2586
2587void AMDGPUCodeGenPassBuilder::addPreRewrite(PassManagerWrapper &PMW) {
2588 if (EnableRegReassign) {
2589 addMachineFunctionPass(Pass: GCNNSAReassignPass(), PMW);
2590 }
2591
2592 addMachineFunctionPass(Pass: AMDGPURewriteAGPRCopyMFMAPass(), PMW);
2593}
2594
2595void AMDGPUCodeGenPassBuilder::addMachineSSAOptimization(
2596 PassManagerWrapper &PMW) {
2597 Base::addMachineSSAOptimization(PMW);
2598
2599 addMachineFunctionPass(Pass: SIFoldOperandsPass(), PMW);
2600 if (EnableDPPCombine) {
2601 addMachineFunctionPass(Pass: GCNDPPCombinePass(), PMW);
2602 }
2603 addMachineFunctionPass(Pass: SILoadStoreOptimizerPass(), PMW);
2604 if (isPassEnabled(Opt: EnableSDWAPeephole)) {
2605 addMachineFunctionPass(Pass: SIPeepholeSDWAPass(), PMW);
2606 addMachineFunctionPass(Pass: EarlyMachineLICMPass(), PMW);
2607 addMachineFunctionPass(Pass: MachineCSEPass(), PMW);
2608 addMachineFunctionPass(Pass: SIFoldOperandsPass(), PMW);
2609 }
2610 addMachineFunctionPass(Pass: DeadMachineInstructionElimPass(), PMW);
2611 addMachineFunctionPass(Pass: SIShrinkInstructionsPass(), PMW);
2612}
2613
2614Error AMDGPUCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
2615 insertPass<PHIEliminationPass>(Pass: SILowerControlFlowPass());
2616
2617 insertPass<TwoAddressInstructionPass>(Pass: SIWholeQuadModePass());
2618
2619 return Base::addFastRegAlloc(PMW);
2620}
2621
2622Error AMDGPUCodeGenPassBuilder::addRegAssignAndRewriteFast(
2623 PassManagerWrapper &PMW) {
2624 if (auto Err = validateRegAllocOptions())
2625 return Err;
2626
2627 addMachineFunctionPass(Pass: GCNPreRALongBranchRegPass(), PMW);
2628
2629 // SGPR allocation - default to fast at -O0.
2630 if (SGPRRegAllocNPM == RegAllocType::Greedy)
2631 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateSGPRs, "sgpr"}), PMW);
2632 else
2633 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateSGPRs, "sgpr", false}),
2634 PMW);
2635
2636 // Equivalent of PEI for SGPRs.
2637 addMachineFunctionPass(Pass: SILowerSGPRSpillsPass(), PMW);
2638
2639 // To Allocate wwm registers used in whole quad mode operations (for shaders).
2640 addMachineFunctionPass(Pass: SIPreAllocateWWMRegsPass(), PMW);
2641
2642 // WWM allocation - default to fast at -O0.
2643 if (WWMRegAllocNPM == RegAllocType::Greedy)
2644 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateWWMRegs, "wwm"}), PMW);
2645 else
2646 addMachineFunctionPass(
2647 Pass: RegAllocFastPass({onlyAllocateWWMRegs, "wwm", false}), PMW);
2648
2649 addMachineFunctionPass(Pass: SILowerWWMCopiesPass(), PMW);
2650 addMachineFunctionPass(Pass: AMDGPUReserveWWMRegsPass(), PMW);
2651
2652 // VGPR allocation - default to fast at -O0.
2653 if (VGPRRegAllocNPM == RegAllocType::Greedy)
2654 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2655 else
2656 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2657
2658 return Error::success();
2659}
2660
2661Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
2662 if (EnableDCEInRA)
2663 insertPass<DetectDeadLanesPass>(Pass: DeadMachineInstructionElimPass());
2664
2665 if (OptVGPRLiveRange)
2666 insertPass<RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>>(
2667 Pass: SIOptimizeVGPRLiveRangePass());
2668
2669 // This must be run immediately after phi elimination and before
2670 // TwoAddressInstructions, otherwise the processing of the tied operand of
2671 // SI_ELSE will introduce a copy of the tied operand source after the else.
2672 insertPass<PHIEliminationPass>(Pass: SILowerControlFlowPass());
2673
2674 if (EnableRewritePartialRegUses)
2675 insertPass<RenameIndependentSubregsPass>(Pass: GCNRewritePartialRegUsesPass());
2676
2677 if (isPassEnabled(Opt: EnablePreRAOptimizations))
2678 insertPass<MachineSchedulerPass>(Pass: GCNPreRAOptimizationsPass());
2679
2680 // Allow the scheduler to run before SIWholeQuadMode inserts exec manipulation
2681 // instructions that cause scheduling barriers.
2682 insertPass<MachineSchedulerPass>(Pass: SIWholeQuadModePass());
2683
2684 if (OptExecMaskPreRA)
2685 insertPass<MachineSchedulerPass>(Pass: SIOptimizeExecMaskingPreRAPass());
2686
2687 // This is not an essential optimization and it has a noticeable impact on
2688 // compilation time, so we only enable it from O2.
2689 if (TM.getOptLevel() > CodeGenOptLevel::Less)
2690 insertPass<MachineSchedulerPass>(Pass: SIFormMemoryClausesPass());
2691
2692 return Base::addOptimizedRegAlloc(PMW);
2693}
2694
2695void AMDGPUCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
2696 if (getOptLevel() != CodeGenOptLevel::None)
2697 addMachineFunctionPass(Pass: AMDGPUPrepareAGPRAllocPass(), PMW);
2698 if (getOptLevel() >= CodeGenOptLevel::Default && EnableMachinePipeliner)
2699 addMachineFunctionPass(Pass: MachinePipelinerPass(), PMW);
2700}
2701
2702Expected<bool> AMDGPUCodeGenPassBuilder::addRegAssignAndRewriteOptimized(
2703 PassManagerWrapper &PMW) {
2704 if (auto Err = validateRegAllocOptions())
2705 return Err;
2706
2707 addMachineFunctionPass(Pass: GCNPreRALongBranchRegPass(), PMW);
2708
2709 // SGPR allocation - default to greedy at -O1 and above.
2710 if (SGPRRegAllocNPM == RegAllocType::Fast)
2711 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateSGPRs, "sgpr", false}),
2712 PMW);
2713 else
2714 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateSGPRs, "sgpr"}), PMW);
2715
2716 // Commit allocated register changes. This is mostly necessary because too
2717 // many things rely on the use lists of the physical registers, such as the
2718 // verifier. This is only necessary with allocators which use LiveIntervals,
2719 // since FastRegAlloc does the replacements itself.
2720 addMachineFunctionPass(Pass: VirtRegRewriterPass(false), PMW);
2721
2722 // At this point, the sgpr-regalloc has been done and it is good to have the
2723 // stack slot coloring to try to optimize the SGPR spill stack indices before
2724 // attempting the custom SGPR spill lowering.
2725 addMachineFunctionPass(Pass: StackSlotColoringPass(), PMW);
2726
2727 // Equivalent of PEI for SGPRs.
2728 addMachineFunctionPass(Pass: SILowerSGPRSpillsPass(), PMW);
2729
2730 // To Allocate wwm registers used in whole quad mode operations (for shaders).
2731 addMachineFunctionPass(Pass: SIPreAllocateWWMRegsPass(), PMW);
2732
2733 // WWM allocation - default to greedy at -O1 and above.
2734 if (WWMRegAllocNPM == RegAllocType::Fast)
2735 addMachineFunctionPass(
2736 Pass: RegAllocFastPass({onlyAllocateWWMRegs, "wwm", false}), PMW);
2737 else
2738 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateWWMRegs, "wwm"}), PMW);
2739 addMachineFunctionPass(Pass: SILowerWWMCopiesPass(), PMW);
2740 addMachineFunctionPass(Pass: VirtRegRewriterPass(false), PMW);
2741 addMachineFunctionPass(Pass: AMDGPUReserveWWMRegsPass(), PMW);
2742
2743 // VGPR allocation - default to greedy at -O1 and above.
2744 if (VGPRRegAllocNPM == RegAllocType::Fast)
2745 addMachineFunctionPass(Pass: RegAllocFastPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2746 else
2747 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateVGPRs, "vgpr"}), PMW);
2748
2749 addPreRewrite(PMW);
2750 addMachineFunctionPass(Pass: VirtRegRewriterPass(true), PMW);
2751
2752 addMachineFunctionPass(Pass: AMDGPUMarkLastScratchLoadPass(), PMW);
2753 return true;
2754}
2755
2756void AMDGPUCodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) {
2757 addMachineFunctionPass(Pass: SIFixVGPRCopiesPass(), PMW);
2758 if (TM.getOptLevel() > CodeGenOptLevel::None)
2759 addMachineFunctionPass(Pass: SIOptimizeExecMaskingPass(), PMW);
2760 Base::addPostRegAlloc(PMW);
2761}
2762
2763void AMDGPUCodeGenPassBuilder::addPreSched2(PassManagerWrapper &PMW) {
2764 if (TM.getOptLevel() > CodeGenOptLevel::None)
2765 addMachineFunctionPass(Pass: SIShrinkInstructionsPass(), PMW);
2766 addMachineFunctionPass(Pass: SIPostRABundlerPass(), PMW);
2767}
2768
2769void AMDGPUCodeGenPassBuilder::addPostBBSections(PassManagerWrapper &PMW) {
2770 // We run this later to avoid passes like livedebugvalues and BBSections
2771 // having to deal with the apparent multi-entry functions we may generate.
2772 addMachineFunctionPass(Pass: AMDGPUPreloadKernArgPrologPass(), PMW);
2773}
2774
2775void AMDGPUCodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
2776 if (isPassEnabled(Opt: EnableVOPD, Level: CodeGenOptLevel::Less)) {
2777 addMachineFunctionPass(Pass: GCNCreateVOPDPass(), PMW);
2778 }
2779
2780 addMachineFunctionPass(Pass: SIMemoryLegalizerPass(), PMW);
2781 if (TM.getOptLevel() > CodeGenOptLevel::None)
2782 addMachineFunctionPass(Pass: SIPostRA16BitMovFoldingPass(), PMW);
2783 addMachineFunctionPass(Pass: SIInsertWaitcntsPass(), PMW);
2784
2785 addMachineFunctionPass(Pass: SIModeRegisterPass(), PMW);
2786
2787 if (TM.getOptLevel() > CodeGenOptLevel::None)
2788 addMachineFunctionPass(Pass: SIInsertHardClausesPass(), PMW);
2789
2790 addMachineFunctionPass(Pass: SILateBranchLoweringPass(), PMW);
2791
2792 if (isPassEnabled(Opt: EnableSetWavePriority, Level: CodeGenOptLevel::Less))
2793 addMachineFunctionPass(Pass: AMDGPUSetWavePriorityPass(), PMW);
2794
2795 if (TM.getOptLevel() > CodeGenOptLevel::None)
2796 addMachineFunctionPass(Pass: SIPreEmitPeepholePass(), PMW);
2797
2798 // The hazard recognizer that runs as part of the post-ra scheduler does not
2799 // guarantee to be able handle all hazards correctly. This is because if there
2800 // are multiple scheduling regions in a basic block, the regions are scheduled
2801 // bottom up, so when we begin to schedule a region we don't know what
2802 // instructions were emitted directly before it.
2803 //
2804 // Here we add a stand-alone hazard recognizer pass which can handle all
2805 // cases.
2806 addMachineFunctionPass(Pass: PostRAHazardRecognizerPass(), PMW);
2807 addMachineFunctionPass(Pass: AMDGPUWaitSGPRHazardsPass(), PMW);
2808 addMachineFunctionPass(Pass: AMDGPULowerVGPREncodingPass(), PMW);
2809
2810 if (isPassEnabled(Opt: EnableInsertDelayAlu, Level: CodeGenOptLevel::Less)) {
2811 addMachineFunctionPass(Pass: AMDGPUInsertDelayAluPass(), PMW);
2812 }
2813
2814 addMachineFunctionPass(Pass: BranchRelaxationPass(), PMW);
2815}
2816
2817bool AMDGPUCodeGenPassBuilder::isPassEnabled(const cl::opt<bool> &Opt,
2818 CodeGenOptLevel Level) const {
2819 if (Opt.getNumOccurrences())
2820 return Opt;
2821 if (TM.getOptLevel() < Level)
2822 return false;
2823 return Opt;
2824}
2825
2826void AMDGPUCodeGenPassBuilder::addEarlyCSEOrGVNPass(PassManagerWrapper &PMW) {
2827 if (TM.getOptLevel() == CodeGenOptLevel::Aggressive)
2828 addFunctionPass(Pass: GVNPass(), PMW);
2829 else
2830 addFunctionPass(Pass: EarlyCSEPass(), PMW);
2831}
2832
2833void AMDGPUCodeGenPassBuilder::addStraightLineScalarOptimizationPasses(
2834 PassManagerWrapper &PMW) {
2835 if (isPassEnabled(Opt: EnableLoopPrefetch, Level: CodeGenOptLevel::Aggressive))
2836 addFunctionPass(Pass: LoopDataPrefetchPass(), PMW);
2837
2838 addFunctionPass(Pass: SeparateConstOffsetFromGEPPass(), PMW);
2839
2840 // ReassociateGEPs exposes more opportunities for SLSR. See
2841 // the example in reassociate-geps-and-slsr.ll.
2842 addFunctionPass(Pass: StraightLineStrengthReducePass(), PMW);
2843
2844 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or
2845 // EarlyCSE can reuse.
2846 addEarlyCSEOrGVNPass(PMW);
2847
2848 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
2849 addFunctionPass(Pass: NaryReassociatePass(), PMW);
2850
2851 // NaryReassociate on GEPs creates redundant common expressions, so run
2852 // EarlyCSE after it.
2853 addFunctionPass(Pass: EarlyCSEPass(), PMW);
2854}
2855