1//===-- AMDGPU.h - MachineFunction passes hw codegen --------------*- C++ -*-=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7/// \file
8//===----------------------------------------------------------------------===//
9
10#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPU_H
11#define LLVM_LIB_TARGET_AMDGPU_AMDGPU_H
12
13#include "llvm/Analysis/CGSCCPassManager.h"
14#include "llvm/CodeGen/MachinePassManager.h"
15#include "llvm/IR/PassManager.h"
16#include "llvm/Pass.h"
17#include "llvm/Support/AMDGPUAddrSpace.h"
18#include "llvm/Support/CodeGen.h"
19
20namespace llvm {
21
22class AMDGPUTargetMachine;
23class LazyCallGraph;
24class GCNTargetMachine;
25class TargetMachine;
26
27// GlobalISel passes
28void initializeAMDGPUPreLegalizerCombinerLegacyPass(PassRegistry &);
29FunctionPass *createAMDGPUPreLegalizeCombinerLegacyPass(bool IsOptLevelNone);
30
31class AMDGPUPreLegalizerCombinerPass
32 : public RequiredPassInfoMixin<AMDGPUPreLegalizerCombinerPass> {
33public:
34 PreservedAnalyses run(MachineFunction &MF,
35 MachineFunctionAnalysisManager &MFAM);
36};
37
38void initializeAMDGPUPostLegalizerCombinerLegacyPass(PassRegistry &);
39FunctionPass *createAMDGPUPostLegalizeCombinerLegacy(bool IsOptNone);
40
41class AMDGPUPostLegalizerCombinerPass
42 : public RequiredPassInfoMixin<AMDGPUPostLegalizerCombinerPass> {
43public:
44 PreservedAnalyses run(MachineFunction &MF,
45 MachineFunctionAnalysisManager &MFAM);
46};
47
48FunctionPass *createAMDGPURegBankCombinerLegacy(bool IsOptLevelNone);
49void initializeAMDGPURegBankCombinerLegacyPass(PassRegistry &);
50
51class AMDGPURegBankCombinerPass
52 : public RequiredPassInfoMixin<AMDGPURegBankCombinerPass> {
53 bool IsOptLevelNone;
54
55public:
56 explicit AMDGPURegBankCombinerPass(bool IsOptLevelNone = false);
57
58 PreservedAnalyses run(MachineFunction &MF,
59 MachineFunctionAnalysisManager &MFAM);
60};
61FunctionPass *createAMDGPUGlobalISelDivergenceLoweringPass();
62
63class AMDGPUGlobalISelDivergenceLoweringPass
64 : public RequiredPassInfoMixin<AMDGPUGlobalISelDivergenceLoweringPass> {
65public:
66 PreservedAnalyses run(MachineFunction &MF,
67 MachineFunctionAnalysisManager &MFAM);
68};
69FunctionPass *createAMDGPURegBankSelectLegacyPass();
70
71class AMDGPURegBankSelectPass
72 : public RequiredPassInfoMixin<AMDGPURegBankSelectPass> {
73public:
74 PreservedAnalyses run(MachineFunction &MF,
75 MachineFunctionAnalysisManager &MFAM);
76
77 MachineFunctionProperties getRequiredProperties() const {
78 return MachineFunctionProperties().setIsSSA().setLegalized();
79 }
80
81 MachineFunctionProperties getSetProperties() const {
82 return MachineFunctionProperties().setRegBankSelected();
83 }
84};
85FunctionPass *createAMDGPURegBankLegalizeLegacyPass();
86
87class AMDGPURegBankLegalizePass
88 : public RequiredPassInfoMixin<AMDGPURegBankLegalizePass> {
89public:
90 PreservedAnalyses run(MachineFunction &MF,
91 MachineFunctionAnalysisManager &MFAM);
92
93 MachineFunctionProperties getRequiredProperties() const {
94 return MachineFunctionProperties()
95 .setIsSSA()
96 .setLegalized()
97 .setRegBankSelected();
98 }
99
100 MachineFunctionProperties getClearedProperties() const {
101 return MachineFunctionProperties().setNoPHIs();
102 }
103};
104
105// SI Passes
106FunctionPass *createGCNDPPCombinePass();
107FunctionPass *createSIAnnotateControlFlowLegacyPass();
108FunctionPass *createSIFoldOperandsLegacyPass();
109FunctionPass *createSIPeepholeSDWALegacyPass();
110FunctionPass *createSILowerI1CopiesLegacyPass();
111FunctionPass *createSIShrinkInstructionsLegacyPass();
112FunctionPass *createSILoadStoreOptimizerLegacyPass();
113FunctionPass *createSIWholeQuadModeLegacyPass();
114FunctionPass *createSIOptimizeExecMaskingPreRAPass();
115FunctionPass *createSIOptimizeVGPRLiveRangeLegacyPass();
116FunctionPass *createAMDGPUNextUseAnalysisLegacyPass();
117FunctionPass *createAMDGPUNextUseAnalysisPrinterLegacyPass();
118FunctionPass *createSIFixSGPRCopiesLegacyPass();
119FunctionPass *createLowerWWMCopiesPass();
120FunctionPass *createSIMemoryLegalizerPass();
121FunctionPass *createSIInsertWaitcntsPass();
122FunctionPass *createSIPreAllocateWWMRegsLegacyPass();
123FunctionPass *createSIFormMemoryClausesLegacyPass();
124
125FunctionPass *createSIPostRABundlerPass();
126FunctionPass *createAMDGPUImageIntrinsicOptimizerPass(const TargetMachine *);
127ModulePass *createAMDGPURemoveIncompatibleFunctionsPass(const TargetMachine *);
128FunctionPass *createAMDGPUCodeGenPreparePass();
129FunctionPass *createAMDGPULateCodeGenPrepareLegacyPass();
130FunctionPass *createAMDGPUReserveWWMRegsPass();
131FunctionPass *createAMDGPURewriteOutArgumentsPass();
132ModulePass *
133createAMDGPULowerModuleLDSLegacyPass(const AMDGPUTargetMachine *TM = nullptr);
134ModulePass *createAMDGPULowerBufferFatPointersPass();
135ModulePass *createAMDGPULowerIntrinsicsLegacyPass();
136FunctionPass *createSIModeRegisterPass();
137FunctionPass *createGCNPreRAOptimizationsLegacyPass();
138FunctionPass *createAMDGPUPreloadKernArgPrologLegacyPass();
139ModulePass *createAMDGPUPreloadKernelArgumentsLegacyPass(const TargetMachine *);
140
141struct AMDGPUSimplifyLibCallsPass
142 : OptionalPassInfoMixin<AMDGPUSimplifyLibCallsPass> {
143 AMDGPUSimplifyLibCallsPass() = default;
144 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
145};
146
147struct AMDGPUImageIntrinsicOptimizerPass
148 : OptionalPassInfoMixin<AMDGPUImageIntrinsicOptimizerPass> {
149 AMDGPUImageIntrinsicOptimizerPass(TargetMachine &TM) : TM(TM) {}
150 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
151
152private:
153 TargetMachine &TM;
154};
155
156struct AMDGPUUseNativeCallsPass
157 : OptionalPassInfoMixin<AMDGPUUseNativeCallsPass> {
158 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
159};
160
161class SILowerI1CopiesPass : public RequiredPassInfoMixin<SILowerI1CopiesPass> {
162public:
163 SILowerI1CopiesPass() = default;
164 PreservedAnalyses run(MachineFunction &MF,
165 MachineFunctionAnalysisManager &MFAM);
166};
167
168void initializeAMDGPUDAGToDAGISelLegacyPass(PassRegistry &);
169
170void initializeAMDGPUAlwaysInlinePass(PassRegistry&);
171
172void initializeAMDGPUAsmPrinterPass(PassRegistry &);
173
174// DPP/Iterative option enables the atomic optimizer with given strategy
175// whereas None disables the atomic optimizer.
176enum class ScanOptions { DPP, Iterative, None };
177FunctionPass *createAMDGPUAtomicOptimizerPass(ScanOptions ScanStrategy);
178void initializeAMDGPUAtomicOptimizerPass(PassRegistry &);
179extern char &AMDGPUAtomicOptimizerID;
180
181ModulePass *createAMDGPUCtorDtorLoweringLegacyPass();
182void initializeAMDGPUCtorDtorLoweringLegacyPass(PassRegistry &);
183extern char &AMDGPUCtorDtorLoweringLegacyPassID;
184
185FunctionPass *createAMDGPULowerKernelArgumentsPass();
186void initializeAMDGPULowerKernelArgumentsPass(PassRegistry &);
187extern char &AMDGPULowerKernelArgumentsID;
188
189FunctionPass *createAMDGPUPromoteKernelArgumentsPass();
190void initializeAMDGPUPromoteKernelArgumentsPass(PassRegistry &);
191extern char &AMDGPUPromoteKernelArgumentsID;
192
193struct AMDGPUPromoteKernelArgumentsPass
194 : OptionalPassInfoMixin<AMDGPUPromoteKernelArgumentsPass> {
195 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
196};
197
198ModulePass *createAMDGPULowerKernelAttributesPass();
199void initializeAMDGPULowerKernelAttributesPass(PassRegistry &);
200extern char &AMDGPULowerKernelAttributesID;
201
202struct AMDGPULowerKernelAttributesPass
203 : OptionalPassInfoMixin<AMDGPULowerKernelAttributesPass> {
204 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
205};
206
207void initializeAMDGPULowerModuleLDSLegacyPass(PassRegistry &);
208extern char &AMDGPULowerModuleLDSLegacyPassID;
209
210struct AMDGPULowerModuleLDSPass
211 : OptionalPassInfoMixin<AMDGPULowerModuleLDSPass> {
212 const AMDGPUTargetMachine &TM;
213 AMDGPULowerModuleLDSPass(const AMDGPUTargetMachine &TM_) : TM(TM_) {}
214
215 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
216};
217
218void initializeAMDGPULowerBufferFatPointersPass(PassRegistry &);
219extern char &AMDGPULowerBufferFatPointersID;
220
221struct AMDGPULowerBufferFatPointersPass
222 : OptionalPassInfoMixin<AMDGPULowerBufferFatPointersPass> {
223 AMDGPULowerBufferFatPointersPass(const TargetMachine &TM) : TM(TM) {}
224 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
225
226private:
227 const TargetMachine &TM;
228};
229
230void initializeAMDGPULowerIntrinsicsLegacyPass(PassRegistry &);
231
232struct AMDGPULowerIntrinsicsPass
233 : OptionalPassInfoMixin<AMDGPULowerIntrinsicsPass> {
234 AMDGPULowerIntrinsicsPass(const AMDGPUTargetMachine &TM) : TM(TM) {}
235 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
236
237private:
238 const AMDGPUTargetMachine &TM;
239};
240
241void initializeAMDGPUPrepareAGPRAllocLegacyPass(PassRegistry &);
242extern char &AMDGPUPrepareAGPRAllocLegacyID;
243
244void initializeAMDGPUReserveWWMRegsLegacyPass(PassRegistry &);
245extern char &AMDGPUReserveWWMRegsLegacyID;
246
247void initializeAMDGPURewriteOutArgumentsPass(PassRegistry &);
248extern char &AMDGPURewriteOutArgumentsID;
249
250void initializeGCNDPPCombineLegacyPass(PassRegistry &);
251extern char &GCNDPPCombineLegacyID;
252
253void initializeSIFoldOperandsLegacyPass(PassRegistry &);
254extern char &SIFoldOperandsLegacyID;
255
256void initializeSIPeepholeSDWALegacyPass(PassRegistry &);
257extern char &SIPeepholeSDWALegacyID;
258
259void initializeSIShrinkInstructionsLegacyPass(PassRegistry &);
260extern char &SIShrinkInstructionsLegacyID;
261
262void initializeSIFixSGPRCopiesLegacyPass(PassRegistry &);
263extern char &SIFixSGPRCopiesLegacyID;
264
265void initializeSIFixVGPRCopiesLegacyPass(PassRegistry &);
266extern char &SIFixVGPRCopiesID;
267
268void initializeAMDGPUNextUseAnalysisLegacyPassPass(PassRegistry &);
269extern char &AMDGPUNextUseAnalysisLegacyID;
270
271void initializeAMDGPUNextUseAnalysisPrinterLegacyPassPass(PassRegistry &);
272extern char &AMDGPUNextUseAnalysisPrinterLegacyID;
273
274void initializeSILowerWWMCopiesLegacyPass(PassRegistry &);
275extern char &SILowerWWMCopiesLegacyID;
276
277void initializeSILowerI1CopiesLegacyPass(PassRegistry &);
278extern char &SILowerI1CopiesLegacyID;
279
280void initializeAMDGPUGlobalISelDivergenceLoweringLegacyPass(PassRegistry &);
281extern char &AMDGPUGlobalISelDivergenceLoweringLegacyID;
282
283void initializeAMDGPURegBankSelectLegacyPass(PassRegistry &);
284extern char &AMDGPURegBankSelectLegacyID;
285
286void initializeAMDGPURegBankLegalizeLegacyPass(PassRegistry &);
287extern char &AMDGPURegBankLegalizeLegacyID;
288
289void initializeAMDGPUMarkLastScratchLoadLegacyPass(PassRegistry &);
290extern char &AMDGPUMarkLastScratchLoadID;
291
292void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &);
293extern char &SILowerSGPRSpillsLegacyID;
294
295void initializeSILoadStoreOptimizerLegacyPass(PassRegistry &);
296extern char &SILoadStoreOptimizerLegacyID;
297
298void initializeSIWholeQuadModeLegacyPass(PassRegistry &);
299extern char &SIWholeQuadModeID;
300
301void initializeSILowerControlFlowLegacyPass(PassRegistry &);
302extern char &SILowerControlFlowLegacyID;
303
304void initializeSIPreEmitPeepholeLegacyPass(PassRegistry &);
305extern char &SIPreEmitPeepholeID;
306
307void initializeSILateBranchLoweringLegacyPass(PassRegistry &);
308extern char &SILateBranchLoweringPassID;
309
310void initializeSIOptimizeExecMaskingLegacyPass(PassRegistry &);
311extern char &SIOptimizeExecMaskingLegacyID;
312
313void initializeSIPreAllocateWWMRegsLegacyPass(PassRegistry &);
314extern char &SIPreAllocateWWMRegsLegacyID;
315
316void initializeAMDGPUImageIntrinsicOptimizerPass(PassRegistry &);
317extern char &AMDGPUImageIntrinsicOptimizerID;
318
319void initializeAMDGPUPerfHintAnalysisLegacyPass(PassRegistry &);
320extern char &AMDGPUPerfHintAnalysisLegacyID;
321
322void initializeGCNRegPressurePrinterPass(PassRegistry &);
323extern char &GCNRegPressurePrinterID;
324
325void initializeAMDGPUPreloadKernArgPrologLegacyPass(PassRegistry &);
326extern char &AMDGPUPreloadKernArgPrologLegacyID;
327
328void initializeAMDGPUPreloadKernelArgumentsLegacyPass(PassRegistry &);
329extern char &AMDGPUPreloadKernelArgumentsLegacyID;
330
331// Passes common to R600 and SI
332FunctionPass *createAMDGPUPromoteAlloca();
333void initializeAMDGPUPromoteAllocaPass(PassRegistry&);
334extern char &AMDGPUPromoteAllocaID;
335
336struct AMDGPUPromoteAllocaPass
337 : OptionalPassInfoMixin<AMDGPUPromoteAllocaPass> {
338 AMDGPUPromoteAllocaPass(TargetMachine &TM) : TM(TM) {}
339 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
340
341private:
342 TargetMachine &TM;
343};
344
345struct AMDGPUPromoteAllocaToVectorPass
346 : OptionalPassInfoMixin<AMDGPUPromoteAllocaToVectorPass> {
347 AMDGPUPromoteAllocaToVectorPass(TargetMachine &TM) : TM(TM) {}
348 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
349
350private:
351 TargetMachine &TM;
352};
353
354struct AMDGPUAtomicOptimizerPass
355 : OptionalPassInfoMixin<AMDGPUAtomicOptimizerPass> {
356 AMDGPUAtomicOptimizerPass(TargetMachine &TM, ScanOptions ScanImpl)
357 : TM(TM), ScanImpl(ScanImpl) {}
358 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
359
360private:
361 TargetMachine &TM;
362 ScanOptions ScanImpl;
363};
364
365struct AMDGPUInsertDelayAluPass
366 : public OptionalPassInfoMixin<AMDGPUInsertDelayAluPass> {
367 PreservedAnalyses run(MachineFunction &F,
368 MachineFunctionAnalysisManager &MFAM);
369};
370
371Pass *createAMDGPUStructurizeCFGPass();
372FunctionPass *createAMDGPUISelDag(TargetMachine &TM, CodeGenOptLevel OptLevel);
373ModulePass *createAMDGPUAlwaysInlinePass(bool GlobalOpt = true);
374
375struct AMDGPUAlwaysInlinePass : OptionalPassInfoMixin<AMDGPUAlwaysInlinePass> {
376 AMDGPUAlwaysInlinePass(bool GlobalOpt = true) : GlobalOpt(GlobalOpt) {}
377 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
378
379private:
380 bool GlobalOpt;
381};
382
383void initializeAMDGPULowerExecSyncLegacyPass(PassRegistry &);
384extern char &AMDGPULowerExecSyncLegacyPassID;
385ModulePass *createAMDGPULowerExecSyncLegacyPass();
386
387struct AMDGPULowerExecSyncPass
388 : OptionalPassInfoMixin<AMDGPULowerExecSyncPass> {
389 AMDGPULowerExecSyncPass() {}
390 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
391};
392
393void initializeAMDGPUSwLowerLDSLegacyPass(PassRegistry &);
394extern char &AMDGPUSwLowerLDSLegacyPassID;
395ModulePass *createAMDGPUSwLowerLDSLegacyPass();
396
397struct AMDGPUSwLowerLDSPass : OptionalPassInfoMixin<AMDGPUSwLowerLDSPass> {
398 AMDGPUSwLowerLDSPass() = default;
399 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
400};
401
402class AMDGPUCodeGenPreparePass
403 : public OptionalPassInfoMixin<AMDGPUCodeGenPreparePass> {
404private:
405 TargetMachine &TM;
406
407public:
408 AMDGPUCodeGenPreparePass(TargetMachine &TM) : TM(TM){};
409 PreservedAnalyses run(Function &, FunctionAnalysisManager &);
410};
411
412class AMDGPULateCodeGenPreparePass
413 : public OptionalPassInfoMixin<AMDGPULateCodeGenPreparePass> {
414private:
415 const GCNTargetMachine &TM;
416
417public:
418 AMDGPULateCodeGenPreparePass(const GCNTargetMachine &TM) : TM(TM) {};
419 PreservedAnalyses run(Function &, FunctionAnalysisManager &);
420};
421
422class AMDGPULowerKernelArgumentsPass
423 : public RequiredPassInfoMixin<AMDGPULowerKernelArgumentsPass> {
424private:
425 TargetMachine &TM;
426
427public:
428 AMDGPULowerKernelArgumentsPass(TargetMachine &TM) : TM(TM){};
429 PreservedAnalyses run(Function &, FunctionAnalysisManager &);
430};
431
432struct AMDGPUAttributorOptions {
433 bool IsClosedWorld = false;
434};
435
436class AMDGPUAttributorPass
437 : public OptionalPassInfoMixin<AMDGPUAttributorPass> {
438private:
439 TargetMachine &TM;
440
441 AMDGPUAttributorOptions Options;
442
443 const ThinOrFullLTOPhase LTOPhase;
444
445public:
446 AMDGPUAttributorPass(TargetMachine &TM, AMDGPUAttributorOptions Options,
447 ThinOrFullLTOPhase LTOPhase = ThinOrFullLTOPhase::None)
448 : TM(TM), Options(Options), LTOPhase(LTOPhase) {};
449 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
450};
451
452class AMDGPUAttributorCGSCCPass
453 : public OptionalPassInfoMixin<AMDGPUAttributorCGSCCPass> {
454private:
455 GCNTargetMachine &TM;
456
457public:
458 AMDGPUAttributorCGSCCPass(GCNTargetMachine &TM) : TM(TM) {}
459 PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
460 LazyCallGraph &CG, CGSCCUpdateResult &UR);
461};
462
463class AMDGPUPreloadKernelArgumentsPass
464 : public OptionalPassInfoMixin<AMDGPUPreloadKernelArgumentsPass> {
465 const TargetMachine &TM;
466
467public:
468 explicit AMDGPUPreloadKernelArgumentsPass(const TargetMachine &TM) : TM(TM) {}
469
470 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
471};
472
473class AMDGPUAnnotateUniformValuesPass
474 : public OptionalPassInfoMixin<AMDGPUAnnotateUniformValuesPass> {
475public:
476 AMDGPUAnnotateUniformValuesPass() = default;
477 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
478};
479
480class SIModeRegisterPass : public RequiredPassInfoMixin<SIModeRegisterPass> {
481public:
482 SIModeRegisterPass() = default;
483 PreservedAnalyses run(MachineFunction &F, MachineFunctionAnalysisManager &AM);
484};
485
486class SIMemoryLegalizerPass
487 : public RequiredPassInfoMixin<SIMemoryLegalizerPass> {
488public:
489 PreservedAnalyses run(MachineFunction &MF,
490 MachineFunctionAnalysisManager &MFAM);
491};
492
493class GCNCreateVOPDPass : public OptionalPassInfoMixin<GCNCreateVOPDPass> {
494public:
495 PreservedAnalyses run(MachineFunction &MF,
496 MachineFunctionAnalysisManager &AM);
497};
498
499class AMDGPUMarkLastScratchLoadPass
500 : public OptionalPassInfoMixin<AMDGPUMarkLastScratchLoadPass> {
501public:
502 PreservedAnalyses run(MachineFunction &MF,
503 MachineFunctionAnalysisManager &AM);
504};
505
506class SIInsertWaitcntsPass
507 : public RequiredPassInfoMixin<SIInsertWaitcntsPass> {
508public:
509 PreservedAnalyses run(MachineFunction &MF,
510 MachineFunctionAnalysisManager &MFAM);
511};
512
513class SIInsertHardClausesPass
514 : public OptionalPassInfoMixin<SIInsertHardClausesPass> {
515public:
516 PreservedAnalyses run(MachineFunction &MF,
517 MachineFunctionAnalysisManager &MFAM);
518};
519
520class SILateBranchLoweringPass
521 : public RequiredPassInfoMixin<SILateBranchLoweringPass> {
522public:
523 PreservedAnalyses run(MachineFunction &MF,
524 MachineFunctionAnalysisManager &MFAM);
525};
526
527class SIPreEmitPeepholePass
528 : public RequiredPassInfoMixin<SIPreEmitPeepholePass> {
529public:
530 PreservedAnalyses run(MachineFunction &MF,
531 MachineFunctionAnalysisManager &MFAM);
532};
533
534class AMDGPUSetWavePriorityPass
535 : public OptionalPassInfoMixin<AMDGPUSetWavePriorityPass> {
536public:
537 PreservedAnalyses run(MachineFunction &MF,
538 MachineFunctionAnalysisManager &MFAM);
539};
540
541FunctionPass *createAMDGPUAnnotateUniformValuesLegacy();
542
543ModulePass *createAMDGPUPrintfRuntimeBinding();
544void initializeAMDGPUPrintfRuntimeBindingPass(PassRegistry&);
545extern char &AMDGPUPrintfRuntimeBindingID;
546
547void initializeAMDGPUResourceUsageAnalysisWrapperPassPass(PassRegistry &);
548extern char &AMDGPUResourceUsageAnalysisID;
549
550struct AMDGPUPrintfRuntimeBindingPass
551 : OptionalPassInfoMixin<AMDGPUPrintfRuntimeBindingPass> {
552 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
553};
554
555void initializeSIOptimizeExecMaskingPreRALegacyPass(PassRegistry &);
556extern char &SIOptimizeExecMaskingPreRAID;
557
558void initializeSIOptimizeVGPRLiveRangeLegacyPass(PassRegistry &);
559extern char &SIOptimizeVGPRLiveRangeLegacyID;
560
561void initializeAMDGPUAnnotateUniformValuesLegacyPass(PassRegistry &);
562extern char &AMDGPUAnnotateUniformValuesLegacyPassID;
563
564void initializeAMDGPUCodeGenPreparePass(PassRegistry&);
565extern char &AMDGPUCodeGenPrepareID;
566
567void initializeAMDGPURemoveIncompatibleFunctionsLegacyPass(PassRegistry &);
568extern char &AMDGPURemoveIncompatibleFunctionsID;
569
570void initializeAMDGPULateCodeGenPrepareLegacyPass(PassRegistry &);
571extern char &AMDGPULateCodeGenPrepareLegacyID;
572
573FunctionPass *createAMDGPURewriteUndefForPHILegacyPass();
574void initializeAMDGPURewriteUndefForPHILegacyPass(PassRegistry &);
575extern char &AMDGPURewriteUndefForPHILegacyPassID;
576
577class AMDGPURewriteUndefForPHIPass
578 : public OptionalPassInfoMixin<AMDGPURewriteUndefForPHIPass> {
579public:
580 AMDGPURewriteUndefForPHIPass() = default;
581 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
582};
583
584class SIAnnotateControlFlowPass
585 : public RequiredPassInfoMixin<SIAnnotateControlFlowPass> {
586private:
587 const AMDGPUTargetMachine &TM;
588
589public:
590 SIAnnotateControlFlowPass(const AMDGPUTargetMachine &TM) : TM(TM) {}
591 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
592};
593
594void initializeSIAnnotateControlFlowLegacyPass(PassRegistry &);
595extern char &SIAnnotateControlFlowLegacyPassID;
596
597void initializeSIMemoryLegalizerLegacyPass(PassRegistry &);
598extern char &SIMemoryLegalizerID;
599
600void initializeSIModeRegisterLegacyPass(PassRegistry &);
601extern char &SIModeRegisterID;
602
603void initializeAMDGPUInsertDelayAluLegacyPass(PassRegistry &);
604extern char &AMDGPUInsertDelayAluID;
605
606void initializeAMDGPULowerVGPREncodingLegacyPass(PassRegistry &);
607extern char &AMDGPULowerVGPREncodingLegacyID;
608
609void initializeSIInsertHardClausesLegacyPass(PassRegistry &);
610extern char &SIInsertHardClausesID;
611
612void initializeSIInsertWaitcntsLegacyPass(PassRegistry &);
613extern char &SIInsertWaitcntsID;
614
615void initializeSIFormMemoryClausesLegacyPass(PassRegistry &);
616extern char &SIFormMemoryClausesID;
617
618void initializeSIPostRABundlerLegacyPass(PassRegistry &);
619extern char &SIPostRABundlerLegacyID;
620
621void initializeGCNCreateVOPDLegacyPass(PassRegistry &);
622extern char &GCNCreateVOPDID;
623
624void initializeAMDGPUUnifyDivergentExitNodesLegacyPass(PassRegistry &);
625extern char &AMDGPUUnifyDivergentExitNodesID;
626
627ImmutablePass *createAMDGPUAAWrapperPass();
628void initializeAMDGPUAAWrapperPassPass(PassRegistry&);
629ImmutablePass *createAMDGPUExternalAAWrapperPass();
630void initializeAMDGPUExternalAAWrapperPass(PassRegistry&);
631
632ModulePass *createAMDGPUExportKernelRuntimeHandlesLegacyPass();
633void initializeAMDGPUExportKernelRuntimeHandlesLegacyPass(PassRegistry &);
634extern char &AMDGPUExportKernelRuntimeHandlesLegacyID;
635
636void initializeGCNNSAReassignLegacyPass(PassRegistry &);
637extern char &GCNNSAReassignID;
638
639void initializeGCNPreRALongBranchRegLegacyPass(PassRegistry &);
640extern char &GCNPreRALongBranchRegID;
641
642void initializeGCNPreRAOptimizationsLegacyPass(PassRegistry &);
643extern char &GCNPreRAOptimizationsID;
644
645FunctionPass *createAMDGPUSetWavePriorityPass();
646void initializeAMDGPUSetWavePriorityLegacyPass(PassRegistry &);
647
648void initializeGCNRewritePartialRegUsesLegacyPass(llvm::PassRegistry &);
649extern char &GCNRewritePartialRegUsesID;
650
651void initializeAMDGPUWaitSGPRHazardsLegacyPass(PassRegistry &);
652extern char &AMDGPUWaitSGPRHazardsLegacyID;
653
654class AMDGPURewriteAGPRCopyMFMAPass
655 : public OptionalPassInfoMixin<AMDGPURewriteAGPRCopyMFMAPass> {
656public:
657 AMDGPURewriteAGPRCopyMFMAPass() = default;
658 PreservedAnalyses run(MachineFunction &MF,
659 MachineFunctionAnalysisManager &MFAM);
660};
661
662void initializeAMDGPURewriteAGPRCopyMFMALegacyPass(PassRegistry &);
663extern char &AMDGPURewriteAGPRCopyMFMALegacyID;
664
665void initializeAMDGPUUniformIntrinsicCombineLegacyPass(PassRegistry &);
666extern char &AMDGPUUniformIntrinsicCombineLegacyPassID;
667FunctionPass *createAMDGPUUniformIntrinsicCombineLegacyPass();
668
669struct AMDGPUUniformIntrinsicCombinePass
670 : public OptionalPassInfoMixin<AMDGPUUniformIntrinsicCombinePass> {
671 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
672};
673
674namespace AMDGPU {
675enum TargetIndex {
676 TI_CONSTDATA_START,
677 TI_SCRATCH_RSRC_DWORD0,
678 TI_SCRATCH_RSRC_DWORD1,
679 TI_SCRATCH_RSRC_DWORD2,
680 TI_SCRATCH_RSRC_DWORD3
681};
682
683static inline bool addrspacesMayAlias(unsigned AS1, unsigned AS2) {
684 if (AS1 > AMDGPUAS::MAX_AMDGPU_ADDRESS || AS2 > AMDGPUAS::MAX_AMDGPU_ADDRESS)
685 return true;
686
687 // clang-format off
688 static const bool ASAliasRules[][AMDGPUAS::MAX_AMDGPU_ADDRESS + 1] = {
689 /* Flat Global Region Local Constant Private Const32 BufFatPtr BufRsrc BufStrdPtr Reserved Reserved Reserved VGPR Reserved Barrier */
690 /* Flat */ {true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false},
691 /* Global */ {true, true, false, false, true, false, true, true, true, true, false, false, false, false, false, false},
692 /* Region */ {false, false, true, false, false, false, false, false, false, false, false, false, false, false, false, false},
693 /* Local */ {true, false, false, true, false, false, false, false, false, false, false, false, false, false, false, false},
694 /* Constant */ {true, true, false, false, false, false, true, true, true, true, false, false, false, false, false, false},
695 /* Private */ {true, false, false, false, false, true, false, false, false, false, false, false, false, false, false, false},
696 /* Constant 32-bit */ {true, true, false, false, true, false, false, true, true, true, false, false, false, false, false, false},
697 /* Buffer Fat Ptr */ {true, true, false, false, true, false, true, true, true, true, false, false, false, false, false, false},
698 /* Buffer Resource */ {true, true, false, false, true, false, true, true, true, true, false, false, false, false, false, false},
699 /* Buffer Strided Ptr */ {true, true, false, false, true, false, true, true, true, true, false, false, false, false, false, false},
700 /* Reserved */ {false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false},
701 /* Reserved */ {false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false},
702 /* Reserved */ {false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false},
703 // A VGPR ("as memory") access only ever touches the wave's own registers,
704 // which no other address space can reach: a flat pointer obtained by casting
705 // one cannot be dereferenced.
706 /* VGPR */ {false, false, false, false, false, false, false, false, false, false, false, false, false, true, false, false},
707 /* Reserved */ {false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false},
708 /* Barrier */ {false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, true},
709 };
710 // clang-format on
711 static_assert(std::size(ASAliasRules) == AMDGPUAS::MAX_AMDGPU_ADDRESS + 1);
712
713 return ASAliasRules[AS1][AS2];
714}
715
716}
717
718} // End namespace llvm
719
720#endif
721