1//===-- AMDGPU.h - MachineFunction passes hw codegen --------------*- C++ -*-=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7/// \file
8//===----------------------------------------------------------------------===//
9
10#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPU_H
11#define LLVM_LIB_TARGET_AMDGPU_AMDGPU_H
12
13#include "llvm/Analysis/CGSCCPassManager.h"
14#include "llvm/CodeGen/MachinePassManager.h"
15#include "llvm/IR/PassManager.h"
16#include "llvm/Pass.h"
17#include "llvm/Support/AMDGPUAddrSpace.h"
18#include "llvm/Support/CodeGen.h"
19
20namespace llvm {
21
22class AMDGPUTargetMachine;
23class LazyCallGraph;
24class GCNTargetMachine;
25class TargetMachine;
26
27// GlobalISel passes
28void initializeAMDGPUPreLegalizerCombinerLegacyPass(PassRegistry &);
29FunctionPass *createAMDGPUPreLegalizeCombinerLegacyPass(bool IsOptLevelNone);
30
31class AMDGPUPreLegalizerCombinerPass
32 : public RequiredPassInfoMixin<AMDGPUPreLegalizerCombinerPass> {
33public:
34 PreservedAnalyses run(MachineFunction &MF,
35 MachineFunctionAnalysisManager &MFAM);
36};
37
38void initializeAMDGPUPostLegalizerCombinerLegacyPass(PassRegistry &);
39FunctionPass *createAMDGPUPostLegalizeCombinerLegacy(bool IsOptNone);
40
41class AMDGPUPostLegalizerCombinerPass
42 : public RequiredPassInfoMixin<AMDGPUPostLegalizerCombinerPass> {
43public:
44 PreservedAnalyses run(MachineFunction &MF,
45 MachineFunctionAnalysisManager &MFAM);
46};
47
48FunctionPass *createAMDGPURegBankCombinerLegacy(bool IsOptLevelNone);
49void initializeAMDGPURegBankCombinerLegacyPass(PassRegistry &);
50
51class AMDGPURegBankCombinerPass
52 : public RequiredPassInfoMixin<AMDGPURegBankCombinerPass> {
53 bool IsOptLevelNone;
54
55public:
56 explicit AMDGPURegBankCombinerPass(bool IsOptLevelNone = false);
57
58 PreservedAnalyses run(MachineFunction &MF,
59 MachineFunctionAnalysisManager &MFAM);
60};
61FunctionPass *createAMDGPUGlobalISelDivergenceLoweringPass();
62
63class AMDGPUGlobalISelDivergenceLoweringPass
64 : public RequiredPassInfoMixin<AMDGPUGlobalISelDivergenceLoweringPass> {
65public:
66 PreservedAnalyses run(MachineFunction &MF,
67 MachineFunctionAnalysisManager &MFAM);
68};
69FunctionPass *createAMDGPURegBankSelectLegacyPass();
70
71class AMDGPURegBankSelectPass
72 : public RequiredPassInfoMixin<AMDGPURegBankSelectPass> {
73public:
74 PreservedAnalyses run(MachineFunction &MF,
75 MachineFunctionAnalysisManager &MFAM);
76
77 MachineFunctionProperties getRequiredProperties() const {
78 return MachineFunctionProperties().setIsSSA().setLegalized();
79 }
80
81 MachineFunctionProperties getSetProperties() const {
82 return MachineFunctionProperties().setRegBankSelected();
83 }
84};
85FunctionPass *createAMDGPURegBankLegalizeLegacyPass();
86
87class AMDGPURegBankLegalizePass
88 : public RequiredPassInfoMixin<AMDGPURegBankLegalizePass> {
89public:
90 PreservedAnalyses run(MachineFunction &MF,
91 MachineFunctionAnalysisManager &MFAM);
92
93 MachineFunctionProperties getRequiredProperties() const {
94 return MachineFunctionProperties()
95 .setIsSSA()
96 .setLegalized()
97 .setRegBankSelected();
98 }
99
100 MachineFunctionProperties getClearedProperties() const {
101 return MachineFunctionProperties().setNoPHIs();
102 }
103};
104
105// SI Passes
106FunctionPass *createSIAnnotateControlFlowLegacyPass();
107FunctionPass *createSILowerI1CopiesLegacyPass();
108FunctionPass *createSIShrinkInstructionsLegacyPass();
109FunctionPass *createSIMemoryLegalizerPass();
110FunctionPass *createSIInsertWaitcntsPass();
111
112FunctionPass *createAMDGPUImageIntrinsicOptimizerPass(const TargetMachine *);
113ModulePass *createAMDGPURemoveIncompatibleFunctionsPass(const TargetMachine *);
114FunctionPass *createAMDGPUCodeGenPreparePass();
115FunctionPass *createAMDGPULateCodeGenPrepareLegacyPass();
116ModulePass *
117createAMDGPULowerModuleLDSLegacyPass(const AMDGPUTargetMachine *TM = nullptr);
118ModulePass *createAMDGPULowerBufferFatPointersPass();
119ModulePass *createAMDGPULowerIntrinsicsLegacyPass();
120FunctionPass *createSIModeRegisterPass();
121FunctionPass *createAMDGPUPreloadKernArgPrologLegacyPass();
122ModulePass *createAMDGPUPreloadKernelArgumentsLegacyPass(const TargetMachine *);
123
124struct AMDGPUSimplifyLibCallsPass
125 : OptionalPassInfoMixin<AMDGPUSimplifyLibCallsPass> {
126 AMDGPUSimplifyLibCallsPass() = default;
127 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
128};
129
130struct AMDGPUImageIntrinsicOptimizerPass
131 : OptionalPassInfoMixin<AMDGPUImageIntrinsicOptimizerPass> {
132 AMDGPUImageIntrinsicOptimizerPass(TargetMachine &TM) : TM(TM) {}
133 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
134
135private:
136 TargetMachine &TM;
137};
138
139struct AMDGPUUseNativeCallsPass
140 : OptionalPassInfoMixin<AMDGPUUseNativeCallsPass> {
141 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
142};
143
144class SILowerI1CopiesPass : public RequiredPassInfoMixin<SILowerI1CopiesPass> {
145public:
146 SILowerI1CopiesPass() = default;
147 PreservedAnalyses run(MachineFunction &MF,
148 MachineFunctionAnalysisManager &MFAM);
149};
150
151void initializeAMDGPUDAGToDAGISelLegacyPass(PassRegistry &);
152
153void initializeAMDGPUAlwaysInlinePass(PassRegistry&);
154
155void initializeAMDGPUAsmPrinterPass(PassRegistry &);
156
157// DPP/Iterative option enables the atomic optimizer with given strategy
158// whereas None disables the atomic optimizer.
159enum class ScanOptions { DPP, Iterative, None };
160FunctionPass *createAMDGPUAtomicOptimizerPass(ScanOptions ScanStrategy);
161void initializeAMDGPUAtomicOptimizerPass(PassRegistry &);
162extern char &AMDGPUAtomicOptimizerID;
163
164ModulePass *createAMDGPUCtorDtorLoweringLegacyPass();
165void initializeAMDGPUCtorDtorLoweringLegacyPass(PassRegistry &);
166extern char &AMDGPUCtorDtorLoweringLegacyPassID;
167
168FunctionPass *createAMDGPULowerKernelArgumentsPass();
169void initializeAMDGPULowerKernelArgumentsPass(PassRegistry &);
170extern char &AMDGPULowerKernelArgumentsID;
171
172void initializeAMDGPUPromoteKernelArgumentsPass(PassRegistry &);
173extern char &AMDGPUPromoteKernelArgumentsID;
174
175struct AMDGPUPromoteKernelArgumentsPass
176 : OptionalPassInfoMixin<AMDGPUPromoteKernelArgumentsPass> {
177 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
178};
179
180void initializeAMDGPULowerKernelAttributesPass(PassRegistry &);
181extern char &AMDGPULowerKernelAttributesID;
182
183struct AMDGPULowerKernelAttributesPass
184 : OptionalPassInfoMixin<AMDGPULowerKernelAttributesPass> {
185 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
186};
187
188void initializeAMDGPULowerModuleLDSLegacyPass(PassRegistry &);
189extern char &AMDGPULowerModuleLDSLegacyPassID;
190
191struct AMDGPULowerModuleLDSPass
192 : OptionalPassInfoMixin<AMDGPULowerModuleLDSPass> {
193 const AMDGPUTargetMachine &TM;
194 AMDGPULowerModuleLDSPass(const AMDGPUTargetMachine &TM_) : TM(TM_) {}
195
196 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
197};
198
199void initializeAMDGPULowerBufferFatPointersPass(PassRegistry &);
200extern char &AMDGPULowerBufferFatPointersID;
201
202struct AMDGPULowerBufferFatPointersPass
203 : OptionalPassInfoMixin<AMDGPULowerBufferFatPointersPass> {
204 AMDGPULowerBufferFatPointersPass(const TargetMachine &TM) : TM(TM) {}
205 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
206
207private:
208 const TargetMachine &TM;
209};
210
211void initializeAMDGPULowerIntrinsicsLegacyPass(PassRegistry &);
212
213struct AMDGPULowerIntrinsicsPass
214 : OptionalPassInfoMixin<AMDGPULowerIntrinsicsPass> {
215 AMDGPULowerIntrinsicsPass(const AMDGPUTargetMachine &TM) : TM(TM) {}
216 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
217
218private:
219 const AMDGPUTargetMachine &TM;
220};
221
222void initializeAMDGPUPrepareAGPRAllocLegacyPass(PassRegistry &);
223extern char &AMDGPUPrepareAGPRAllocLegacyID;
224
225void initializeAMDGPUReserveWWMRegsLegacyPass(PassRegistry &);
226extern char &AMDGPUReserveWWMRegsLegacyID;
227
228void initializeAMDGPURewriteOutArgumentsPass(PassRegistry &);
229extern char &AMDGPURewriteOutArgumentsID;
230
231void initializeGCNDPPCombineLegacyPass(PassRegistry &);
232extern char &GCNDPPCombineLegacyID;
233
234void initializeSIFoldOperandsLegacyPass(PassRegistry &);
235extern char &SIFoldOperandsLegacyID;
236
237void initializeSIPeepholeSDWALegacyPass(PassRegistry &);
238extern char &SIPeepholeSDWALegacyID;
239
240void initializeSIShrinkInstructionsLegacyPass(PassRegistry &);
241extern char &SIShrinkInstructionsLegacyID;
242
243void initializeSIFixSGPRCopiesLegacyPass(PassRegistry &);
244extern char &SIFixSGPRCopiesLegacyID;
245
246void initializeSIFixVGPRCopiesLegacyPass(PassRegistry &);
247extern char &SIFixVGPRCopiesID;
248
249void initializeAMDGPUNextUseAnalysisLegacyPassPass(PassRegistry &);
250extern char &AMDGPUNextUseAnalysisLegacyID;
251
252void initializeAMDGPUNextUseAnalysisPrinterLegacyPassPass(PassRegistry &);
253extern char &AMDGPUNextUseAnalysisPrinterLegacyID;
254
255void initializeSILowerWWMCopiesLegacyPass(PassRegistry &);
256extern char &SILowerWWMCopiesLegacyID;
257
258void initializeSILowerI1CopiesLegacyPass(PassRegistry &);
259extern char &SILowerI1CopiesLegacyID;
260
261void initializeAMDGPUGlobalISelDivergenceLoweringLegacyPass(PassRegistry &);
262extern char &AMDGPUGlobalISelDivergenceLoweringLegacyID;
263
264void initializeAMDGPURegBankSelectLegacyPass(PassRegistry &);
265extern char &AMDGPURegBankSelectLegacyID;
266
267void initializeAMDGPURegBankLegalizeLegacyPass(PassRegistry &);
268extern char &AMDGPURegBankLegalizeLegacyID;
269
270void initializeAMDGPUMarkLastScratchLoadLegacyPass(PassRegistry &);
271extern char &AMDGPUMarkLastScratchLoadID;
272
273void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &);
274extern char &SILowerSGPRSpillsLegacyID;
275
276void initializeSILoadStoreOptimizerLegacyPass(PassRegistry &);
277extern char &SILoadStoreOptimizerLegacyID;
278
279void initializeSIWholeQuadModeLegacyPass(PassRegistry &);
280extern char &SIWholeQuadModeID;
281
282void initializeSILowerControlFlowLegacyPass(PassRegistry &);
283extern char &SILowerControlFlowLegacyID;
284
285void initializeSIPreEmitPeepholeLegacyPass(PassRegistry &);
286extern char &SIPreEmitPeepholeID;
287
288void initializeSILateBranchLoweringLegacyPass(PassRegistry &);
289extern char &SILateBranchLoweringPassID;
290
291void initializeSIOptimizeExecMaskingLegacyPass(PassRegistry &);
292extern char &SIOptimizeExecMaskingLegacyID;
293
294void initializeSIPreAllocateWWMRegsLegacyPass(PassRegistry &);
295extern char &SIPreAllocateWWMRegsLegacyID;
296
297void initializeAMDGPUImageIntrinsicOptimizerPass(PassRegistry &);
298extern char &AMDGPUImageIntrinsicOptimizerID;
299
300void initializeAMDGPUPerfHintAnalysisLegacyPass(PassRegistry &);
301extern char &AMDGPUPerfHintAnalysisLegacyID;
302
303void initializeGCNRegPressurePrinterPass(PassRegistry &);
304extern char &GCNRegPressurePrinterID;
305
306void initializeAMDGPUPreloadKernArgPrologLegacyPass(PassRegistry &);
307extern char &AMDGPUPreloadKernArgPrologLegacyID;
308
309void initializeAMDGPUPreloadKernelArgumentsLegacyPass(PassRegistry &);
310extern char &AMDGPUPreloadKernelArgumentsLegacyID;
311
312void initializeSIPostRA16BitMovFoldingLegacyPass(PassRegistry &);
313extern char &SIPostRA16BitMovFoldingLegacyID;
314
315// Passes common to R600 and SI
316FunctionPass *createAMDGPUPromoteAlloca();
317void initializeAMDGPUPromoteAllocaPass(PassRegistry&);
318extern char &AMDGPUPromoteAllocaID;
319
320struct AMDGPUPromoteAllocaPass
321 : OptionalPassInfoMixin<AMDGPUPromoteAllocaPass> {
322 AMDGPUPromoteAllocaPass(TargetMachine &TM) : TM(TM) {}
323 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
324
325private:
326 TargetMachine &TM;
327};
328
329struct AMDGPUPromoteAllocaToVectorPass
330 : OptionalPassInfoMixin<AMDGPUPromoteAllocaToVectorPass> {
331 AMDGPUPromoteAllocaToVectorPass(TargetMachine &TM) : TM(TM) {}
332 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
333
334private:
335 TargetMachine &TM;
336};
337
338struct AMDGPUAtomicOptimizerPass
339 : OptionalPassInfoMixin<AMDGPUAtomicOptimizerPass> {
340 AMDGPUAtomicOptimizerPass(TargetMachine &TM, ScanOptions ScanImpl)
341 : TM(TM), ScanImpl(ScanImpl) {}
342 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
343
344private:
345 TargetMachine &TM;
346 ScanOptions ScanImpl;
347};
348
349struct AMDGPUInsertDelayAluPass
350 : public OptionalPassInfoMixin<AMDGPUInsertDelayAluPass> {
351 PreservedAnalyses run(MachineFunction &F,
352 MachineFunctionAnalysisManager &MFAM);
353};
354
355FunctionPass *createAMDGPUISelDag(TargetMachine &TM, CodeGenOptLevel OptLevel);
356ModulePass *createAMDGPUAlwaysInlinePass(bool GlobalOpt = true);
357
358struct AMDGPUAlwaysInlinePass : OptionalPassInfoMixin<AMDGPUAlwaysInlinePass> {
359 AMDGPUAlwaysInlinePass(bool GlobalOpt = true) : GlobalOpt(GlobalOpt) {}
360 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
361
362private:
363 bool GlobalOpt;
364};
365
366void initializeAMDGPULowerExecSyncLegacyPass(PassRegistry &);
367extern char &AMDGPULowerExecSyncLegacyPassID;
368ModulePass *createAMDGPULowerExecSyncLegacyPass();
369
370struct AMDGPULowerExecSyncPass
371 : OptionalPassInfoMixin<AMDGPULowerExecSyncPass> {
372 AMDGPULowerExecSyncPass() {}
373 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
374};
375
376void initializeAMDGPUSwLowerLDSLegacyPass(PassRegistry &);
377extern char &AMDGPUSwLowerLDSLegacyPassID;
378ModulePass *createAMDGPUSwLowerLDSLegacyPass();
379
380struct AMDGPUSwLowerLDSPass : OptionalPassInfoMixin<AMDGPUSwLowerLDSPass> {
381 AMDGPUSwLowerLDSPass() = default;
382 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
383};
384
385class AMDGPUCodeGenPreparePass
386 : public OptionalPassInfoMixin<AMDGPUCodeGenPreparePass> {
387private:
388 TargetMachine &TM;
389
390public:
391 AMDGPUCodeGenPreparePass(TargetMachine &TM) : TM(TM){};
392 PreservedAnalyses run(Function &, FunctionAnalysisManager &);
393};
394
395class AMDGPULateCodeGenPreparePass
396 : public OptionalPassInfoMixin<AMDGPULateCodeGenPreparePass> {
397private:
398 const GCNTargetMachine &TM;
399
400public:
401 AMDGPULateCodeGenPreparePass(const GCNTargetMachine &TM) : TM(TM) {};
402 PreservedAnalyses run(Function &, FunctionAnalysisManager &);
403};
404
405class AMDGPULowerKernelArgumentsPass
406 : public RequiredPassInfoMixin<AMDGPULowerKernelArgumentsPass> {
407private:
408 TargetMachine &TM;
409
410public:
411 AMDGPULowerKernelArgumentsPass(TargetMachine &TM) : TM(TM){};
412 PreservedAnalyses run(Function &, FunctionAnalysisManager &);
413};
414
415struct AMDGPUAttributorOptions {
416 bool IsClosedWorld = false;
417};
418
419class AMDGPUAttributorPass
420 : public OptionalPassInfoMixin<AMDGPUAttributorPass> {
421private:
422 TargetMachine &TM;
423
424 AMDGPUAttributorOptions Options;
425
426 const ThinOrFullLTOPhase LTOPhase;
427
428public:
429 AMDGPUAttributorPass(TargetMachine &TM, AMDGPUAttributorOptions Options,
430 ThinOrFullLTOPhase LTOPhase = ThinOrFullLTOPhase::None)
431 : TM(TM), Options(Options), LTOPhase(LTOPhase) {};
432 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
433};
434
435class AMDGPUAttributorCGSCCPass
436 : public OptionalPassInfoMixin<AMDGPUAttributorCGSCCPass> {
437private:
438 GCNTargetMachine &TM;
439
440public:
441 AMDGPUAttributorCGSCCPass(GCNTargetMachine &TM) : TM(TM) {}
442 PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
443 LazyCallGraph &CG, CGSCCUpdateResult &UR);
444};
445
446class AMDGPUPreloadKernelArgumentsPass
447 : public OptionalPassInfoMixin<AMDGPUPreloadKernelArgumentsPass> {
448 const TargetMachine &TM;
449
450public:
451 explicit AMDGPUPreloadKernelArgumentsPass(const TargetMachine &TM) : TM(TM) {}
452
453 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
454};
455
456class AMDGPUAnnotateUniformValuesPass
457 : public OptionalPassInfoMixin<AMDGPUAnnotateUniformValuesPass> {
458public:
459 AMDGPUAnnotateUniformValuesPass() = default;
460 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
461};
462
463class SIModeRegisterPass : public RequiredPassInfoMixin<SIModeRegisterPass> {
464public:
465 SIModeRegisterPass() = default;
466 PreservedAnalyses run(MachineFunction &F, MachineFunctionAnalysisManager &AM);
467};
468
469class SIMemoryLegalizerPass
470 : public RequiredPassInfoMixin<SIMemoryLegalizerPass> {
471public:
472 PreservedAnalyses run(MachineFunction &MF,
473 MachineFunctionAnalysisManager &MFAM);
474};
475
476class GCNCreateVOPDPass : public OptionalPassInfoMixin<GCNCreateVOPDPass> {
477public:
478 PreservedAnalyses run(MachineFunction &MF,
479 MachineFunctionAnalysisManager &AM);
480};
481
482class AMDGPUMarkLastScratchLoadPass
483 : public OptionalPassInfoMixin<AMDGPUMarkLastScratchLoadPass> {
484public:
485 PreservedAnalyses run(MachineFunction &MF,
486 MachineFunctionAnalysisManager &AM);
487};
488
489class SIInsertWaitcntsPass
490 : public RequiredPassInfoMixin<SIInsertWaitcntsPass> {
491public:
492 PreservedAnalyses run(MachineFunction &MF,
493 MachineFunctionAnalysisManager &MFAM);
494};
495
496class SIInsertHardClausesPass
497 : public OptionalPassInfoMixin<SIInsertHardClausesPass> {
498public:
499 PreservedAnalyses run(MachineFunction &MF,
500 MachineFunctionAnalysisManager &MFAM);
501};
502
503class SILateBranchLoweringPass
504 : public RequiredPassInfoMixin<SILateBranchLoweringPass> {
505public:
506 PreservedAnalyses run(MachineFunction &MF,
507 MachineFunctionAnalysisManager &MFAM);
508};
509
510class SIPreEmitPeepholePass
511 : public RequiredPassInfoMixin<SIPreEmitPeepholePass> {
512public:
513 PreservedAnalyses run(MachineFunction &MF,
514 MachineFunctionAnalysisManager &MFAM);
515};
516
517class AMDGPUSetWavePriorityPass
518 : public OptionalPassInfoMixin<AMDGPUSetWavePriorityPass> {
519public:
520 PreservedAnalyses run(MachineFunction &MF,
521 MachineFunctionAnalysisManager &MFAM);
522};
523
524class SIPostRA16BitMovFoldingPass
525 : public OptionalPassInfoMixin<SIPostRA16BitMovFoldingPass> {
526public:
527 PreservedAnalyses run(MachineFunction &MF,
528 MachineFunctionAnalysisManager &MFAM);
529};
530
531FunctionPass *createAMDGPUAnnotateUniformValuesLegacy();
532
533ModulePass *createAMDGPUPrintfRuntimeBinding();
534void initializeAMDGPUPrintfRuntimeBindingPass(PassRegistry&);
535extern char &AMDGPUPrintfRuntimeBindingID;
536
537void initializeAMDGPUResourceUsageAnalysisWrapperPassPass(PassRegistry &);
538extern char &AMDGPUResourceUsageAnalysisID;
539
540struct AMDGPUPrintfRuntimeBindingPass
541 : OptionalPassInfoMixin<AMDGPUPrintfRuntimeBindingPass> {
542 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
543};
544
545void initializeSIOptimizeExecMaskingPreRALegacyPass(PassRegistry &);
546extern char &SIOptimizeExecMaskingPreRAID;
547
548void initializeSIOptimizeVGPRLiveRangeLegacyPass(PassRegistry &);
549extern char &SIOptimizeVGPRLiveRangeLegacyID;
550
551void initializeAMDGPUAnnotateUniformValuesLegacyPass(PassRegistry &);
552extern char &AMDGPUAnnotateUniformValuesLegacyPassID;
553
554void initializeAMDGPUCodeGenPreparePass(PassRegistry&);
555extern char &AMDGPUCodeGenPrepareID;
556
557void initializeAMDGPURemoveIncompatibleFunctionsLegacyPass(PassRegistry &);
558extern char &AMDGPURemoveIncompatibleFunctionsID;
559
560void initializeAMDGPULateCodeGenPrepareLegacyPass(PassRegistry &);
561extern char &AMDGPULateCodeGenPrepareLegacyID;
562
563FunctionPass *createAMDGPURewriteUndefForPHILegacyPass();
564void initializeAMDGPURewriteUndefForPHILegacyPass(PassRegistry &);
565extern char &AMDGPURewriteUndefForPHILegacyPassID;
566
567class AMDGPURewriteUndefForPHIPass
568 : public OptionalPassInfoMixin<AMDGPURewriteUndefForPHIPass> {
569public:
570 AMDGPURewriteUndefForPHIPass() = default;
571 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
572};
573
574class SIAnnotateControlFlowPass
575 : public RequiredPassInfoMixin<SIAnnotateControlFlowPass> {
576private:
577 const AMDGPUTargetMachine &TM;
578
579public:
580 SIAnnotateControlFlowPass(const AMDGPUTargetMachine &TM) : TM(TM) {}
581 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
582};
583
584void initializeSIAnnotateControlFlowLegacyPass(PassRegistry &);
585extern char &SIAnnotateControlFlowLegacyPassID;
586
587void initializeSIMemoryLegalizerLegacyPass(PassRegistry &);
588extern char &SIMemoryLegalizerID;
589
590void initializeSIModeRegisterLegacyPass(PassRegistry &);
591extern char &SIModeRegisterID;
592
593void initializeAMDGPUInsertDelayAluLegacyPass(PassRegistry &);
594extern char &AMDGPUInsertDelayAluID;
595
596void initializeAMDGPULowerVGPREncodingLegacyPass(PassRegistry &);
597extern char &AMDGPULowerVGPREncodingLegacyID;
598
599void initializeSIInsertHardClausesLegacyPass(PassRegistry &);
600extern char &SIInsertHardClausesID;
601
602void initializeSIInsertWaitcntsLegacyPass(PassRegistry &);
603extern char &SIInsertWaitcntsID;
604
605void initializeSIFormMemoryClausesLegacyPass(PassRegistry &);
606extern char &SIFormMemoryClausesID;
607
608void initializeSIPostRABundlerLegacyPass(PassRegistry &);
609extern char &SIPostRABundlerLegacyID;
610
611void initializeGCNCreateVOPDLegacyPass(PassRegistry &);
612extern char &GCNCreateVOPDID;
613
614void initializeAMDGPUUnifyDivergentExitNodesLegacyPass(PassRegistry &);
615extern char &AMDGPUUnifyDivergentExitNodesID;
616
617ImmutablePass *createAMDGPUAAWrapperPass();
618void initializeAMDGPUAAWrapperPassPass(PassRegistry &);
619void initializeAMDGPUExternalAAWrapperPass(PassRegistry&);
620
621ModulePass *createAMDGPUExportKernelRuntimeHandlesLegacyPass();
622void initializeAMDGPUExportKernelRuntimeHandlesLegacyPass(PassRegistry &);
623extern char &AMDGPUExportKernelRuntimeHandlesLegacyID;
624
625void initializeGCNNSAReassignLegacyPass(PassRegistry &);
626extern char &GCNNSAReassignID;
627
628void initializeGCNPreRALongBranchRegLegacyPass(PassRegistry &);
629extern char &GCNPreRALongBranchRegID;
630
631void initializeGCNPreRAOptimizationsLegacyPass(PassRegistry &);
632extern char &GCNPreRAOptimizationsID;
633
634FunctionPass *createAMDGPUSetWavePriorityPass();
635void initializeAMDGPUSetWavePriorityLegacyPass(PassRegistry &);
636
637void initializeGCNRewritePartialRegUsesLegacyPass(llvm::PassRegistry &);
638extern char &GCNRewritePartialRegUsesID;
639
640void initializeAMDGPUWaitSGPRHazardsLegacyPass(PassRegistry &);
641extern char &AMDGPUWaitSGPRHazardsLegacyID;
642
643class AMDGPURewriteAGPRCopyMFMAPass
644 : public OptionalPassInfoMixin<AMDGPURewriteAGPRCopyMFMAPass> {
645public:
646 AMDGPURewriteAGPRCopyMFMAPass() = default;
647 PreservedAnalyses run(MachineFunction &MF,
648 MachineFunctionAnalysisManager &MFAM);
649};
650
651void initializeAMDGPURewriteAGPRCopyMFMALegacyPass(PassRegistry &);
652extern char &AMDGPURewriteAGPRCopyMFMALegacyID;
653
654void initializeAMDGPUUniformIntrinsicCombineLegacyPass(PassRegistry &);
655extern char &AMDGPUUniformIntrinsicCombineLegacyPassID;
656FunctionPass *createAMDGPUUniformIntrinsicCombineLegacyPass();
657
658struct AMDGPUUniformIntrinsicCombinePass
659 : public OptionalPassInfoMixin<AMDGPUUniformIntrinsicCombinePass> {
660 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
661};
662
663namespace AMDGPU {
664enum TargetIndex {
665 TI_CONSTDATA_START,
666 TI_SCRATCH_RSRC_DWORD0,
667 TI_SCRATCH_RSRC_DWORD1,
668 TI_SCRATCH_RSRC_DWORD2,
669 TI_SCRATCH_RSRC_DWORD3
670};
671
672static inline bool addrspacesMayAlias(unsigned AS1, unsigned AS2) {
673 if (AS1 > AMDGPUAS::MAX_AMDGPU_ADDRESS || AS2 > AMDGPUAS::MAX_AMDGPU_ADDRESS)
674 return true;
675
676 // clang-format off
677 static const bool ASAliasRules[][AMDGPUAS::MAX_AMDGPU_ADDRESS + 1] = {
678 /* Flat Global Region Local Constant Private Const32 BufFatPtr BufRsrc BufStrdPtr Reserved Reserved Reserved VGPR Reserved Barrier */
679 /* Flat */ {true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false},
680 /* Global */ {true, true, false, false, true, false, true, true, true, true, false, false, false, false, false, false},
681 /* Region */ {false, false, true, false, false, false, false, false, false, false, false, false, false, false, false, false},
682 /* Local */ {true, false, false, true, false, false, false, false, false, false, false, false, false, false, false, false},
683 /* Constant */ {true, true, false, false, false, false, true, true, true, true, false, false, false, false, false, false},
684 /* Private */ {true, false, false, false, false, true, false, false, false, false, false, false, false, false, false, false},
685 /* Constant 32-bit */ {true, true, false, false, true, false, false, true, true, true, false, false, false, false, false, false},
686 /* Buffer Fat Ptr */ {true, true, false, false, true, false, true, true, true, true, false, false, false, false, false, false},
687 /* Buffer Resource */ {true, true, false, false, true, false, true, true, true, true, false, false, false, false, false, false},
688 /* Buffer Strided Ptr */ {true, true, false, false, true, false, true, true, true, true, false, false, false, false, false, false},
689 /* Reserved */ {false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false},
690 /* Reserved */ {false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false},
691 /* Reserved */ {false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false},
692 // A VGPR ("as memory") access only ever touches the wave's own registers,
693 // which no other address space can reach: a flat pointer obtained by casting
694 // one cannot be dereferenced.
695 /* VGPR */ {false, false, false, false, false, false, false, false, false, false, false, false, false, true, false, false},
696 /* Reserved */ {false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false},
697 /* Barrier */ {false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true},
698 };
699 // clang-format on
700 static_assert(std::size(ASAliasRules) == AMDGPUAS::MAX_AMDGPU_ADDRESS + 1);
701
702 return ASAliasRules[AS1][AS2];
703}
704
705}
706
707} // End namespace llvm
708
709#endif
710