1//===-- AMDGPURegBankSelect.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// Assign register banks to all register operands of G_ instructions using
10/// machine uniformity analysis.
11/// Sgpr - uniform values and some lane masks
12/// Vgpr - divergent, non S1, values
13/// Vcc - divergent S1 values(lane masks)
14/// However in some cases G_ instructions with this register bank assignment
15/// can't be inst-selected. This is solved in AMDGPURegBankLegalize.
16//===----------------------------------------------------------------------===//
17
18#include "AMDGPU.h"
19#include "AMDGPUGlobalISelUtils.h"
20#include "GCNSubtarget.h"
21#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
22#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
23#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
24#include "llvm/CodeGen/MachinePassManager.h"
25#include "llvm/CodeGen/MachineUniformityAnalysis.h"
26#include "llvm/CodeGen/TargetPassConfig.h"
27#include "llvm/InitializePasses.h"
28
29#define DEBUG_TYPE "amdgpu-reg-bank-select"
30
31using namespace llvm;
32using namespace AMDGPU;
33
34namespace {
35
36class AMDGPURegBankSelectLegacy : public MachineFunctionPass {
37public:
38 static char ID;
39
40 AMDGPURegBankSelectLegacy() : MachineFunctionPass(ID) {}
41
42 bool runOnMachineFunction(MachineFunction &MF) override;
43
44 StringRef getPassName() const override {
45 return "AMDGPU Register Bank Select";
46 }
47
48 void getAnalysisUsage(AnalysisUsage &AU) const override {
49 AU.addRequired<TargetPassConfig>();
50 AU.addRequired<GISelCSEAnalysisWrapperPass>();
51 AU.addRequired<MachineUniformityAnalysisPass>();
52 MachineFunctionPass::getAnalysisUsage(AU);
53 }
54
55 // This pass assigns register banks to all virtual registers, and we maintain
56 // this property in subsequent passes
57 MachineFunctionProperties getSetProperties() const override {
58 return MachineFunctionProperties().setRegBankSelected();
59 }
60};
61
62} // End anonymous namespace.
63
64INITIALIZE_PASS_BEGIN(AMDGPURegBankSelectLegacy, DEBUG_TYPE,
65 "AMDGPU Register Bank Select", false, false)
66INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
67INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
68INITIALIZE_PASS_DEPENDENCY(MachineUniformityAnalysisPass)
69INITIALIZE_PASS_END(AMDGPURegBankSelectLegacy, DEBUG_TYPE,
70 "AMDGPU Register Bank Select", false, false)
71
72char AMDGPURegBankSelectLegacy::ID = 0;
73
74char &llvm::AMDGPURegBankSelectLegacyID = AMDGPURegBankSelectLegacy::ID;
75
76FunctionPass *llvm::createAMDGPURegBankSelectLegacyPass() {
77 return new AMDGPURegBankSelectLegacy();
78}
79
80class RegBankSelectHelper {
81 MachineIRBuilder &B;
82 MachineRegisterInfo &MRI;
83 AMDGPU::IntrinsicLaneMaskAnalyzer &ILMA;
84 const MachineUniformityInfo &MUI;
85 const SIRegisterInfo &TRI;
86 const RegisterBank *SgprRB;
87 const RegisterBank *VgprRB;
88 const RegisterBank *VccRB;
89
90public:
91 RegBankSelectHelper(MachineIRBuilder &B,
92 AMDGPU::IntrinsicLaneMaskAnalyzer &ILMA,
93 const MachineUniformityInfo &MUI,
94 const SIRegisterInfo &TRI, const RegisterBankInfo &RBI)
95 : B(B), MRI(*B.getMRI()), ILMA(ILMA), MUI(MUI), TRI(TRI),
96 SgprRB(&RBI.getRegBank(ID: AMDGPU::SGPRRegBankID)),
97 VgprRB(&RBI.getRegBank(ID: AMDGPU::VGPRRegBankID)),
98 VccRB(&RBI.getRegBank(ID: AMDGPU::VCCRegBankID)) {}
99
100 // Temporal divergence copy: COPY to vgpr with implicit use of $exec inside of
101 // the cycle
102 // Note: uniformity analysis does not consider that registers with vgpr def
103 // are divergent (you can have uniform value in vgpr).
104 // - TODO: implicit use of $exec could be implemented as indicator that
105 // instruction is divergent
106 bool isTemporalDivergenceCopy(Register Reg) {
107 MachineInstr *MI = MRI.getVRegDef(Reg);
108 if (!MI->isCopy() || MI->getNumImplicitOperands() != 1)
109 return false;
110
111 return MI->implicit_operands().begin()->getReg() == TRI.getExec();
112 }
113
114 const RegisterBank *getRegBankToAssign(Register Reg) {
115 if (!isTemporalDivergenceCopy(Reg) &&
116 (MUI.isUniformAtDef(V: Reg) || ILMA.isS32S64LaneMask(Reg)))
117 return SgprRB;
118 if (MRI.getType(Reg) == LLT::scalar(SizeInBits: 1))
119 return VccRB;
120 return VgprRB;
121 }
122
123 // %rc:RegClass(s32) = G_ ...
124 // ...
125 // %a = G_ ..., %rc
126 // ->
127 // %rb:RegBank(s32) = G_ ...
128 // %rc:RegClass(s32) = COPY %rb
129 // ...
130 // %a = G_ ..., %rb
131 void reAssignRegBankOnDef(MachineInstr &MI, MachineOperand &DefOP,
132 const RegisterBank *RB) {
133 // Register that already has Register class got it during pre-inst selection
134 // of another instruction. Maybe cross bank copy was required so we insert a
135 // copy that can be removed later. This simplifies post regbanklegalize
136 // combiner and avoids need to special case some patterns.
137 Register Reg = DefOP.getReg();
138 LLT Ty = MRI.getType(Reg);
139 Register NewReg = MRI.createVirtualRegister(RegAttr: {.RCOrRB: RB, .Ty: Ty});
140 DefOP.setReg(NewReg);
141
142 auto &MBB = *MI.getParent();
143 B.setInsertPt(MBB, II: MBB.SkipPHIsAndLabels(I: std::next(x: MI.getIterator())));
144 B.buildCopy(Res: Reg, Op: NewReg);
145
146 // The problem was discovered for uniform S1 that was used as both
147 // lane mask(vcc) and regular sgpr S1.
148 // - lane-mask(vcc) use was by si_if, this use is divergent and requires
149 // non-trivial sgpr-S1-to-vcc copy. But pre-inst-selection of si_if sets
150 // sreg_64_xexec(S1) on def of uniform S1 making it lane-mask.
151 // - the regular sgpr S1(uniform) instruction is now broken since
152 // it uses sreg_64_xexec(S1) which is divergent.
153
154 // Replace virtual registers with register class on generic instructions
155 // uses with virtual registers with register bank.
156 for (auto &UseMI : make_early_inc_range(Range: MRI.use_instructions(Reg))) {
157 if (UseMI.isPreISelOpcode()) {
158 for (MachineOperand &Op : UseMI.operands()) {
159 if (Op.isReg() && Op.getReg() == Reg)
160 Op.setReg(NewReg);
161 }
162 }
163 }
164 }
165
166 // %a = G_ ..., %rc
167 // ->
168 // %rb:RegBank(s32) = COPY %rc
169 // %a = G_ ..., %rb
170 void constrainRegBankUse(MachineInstr &MI, MachineOperand &UseOP,
171 const RegisterBank *RB) {
172 Register Reg = UseOP.getReg();
173
174 LLT Ty = MRI.getType(Reg);
175 Register NewReg = MRI.createVirtualRegister(RegAttr: {.RCOrRB: RB, .Ty: Ty});
176 UseOP.setReg(NewReg);
177
178 if (MI.isPHI()) {
179 auto DefMI = MRI.getVRegDef(Reg)->getIterator();
180 MachineBasicBlock *DefMBB = DefMI->getParent();
181 B.setInsertPt(MBB&: *DefMBB, II: DefMBB->SkipPHIsAndLabels(I: std::next(x: DefMI)));
182 } else {
183 B.setInstr(MI);
184 }
185
186 B.buildCopy(Res: NewReg, Op: Reg);
187 }
188};
189
190static Register getVReg(MachineOperand &Op) {
191 if (!Op.isReg())
192 return {};
193
194 // Operands of COPY and G_SI_CALL can be physical registers.
195 Register Reg = Op.getReg();
196 if (!Reg.isVirtual())
197 return {};
198
199 return Reg;
200}
201
202static bool
203runRegBankSelect(MachineFunction &MF, function_ref<GISelCSEInfo *()> GetCSEInfo,
204 function_ref<const MachineUniformityInfo *()> GetMUI) {
205 if (MF.getProperties().hasFailedISel())
206 return false;
207
208 GISelCSEInfo &CSEInfo = *GetCSEInfo();
209 const MachineUniformityInfo &MUI = *GetMUI();
210
211 // Setup the instruction builder with CSE.
212 GISelObserverWrapper Observer;
213 Observer.addObserver(O: &CSEInfo);
214
215 CSEMIRBuilder B(MF);
216 B.setCSEInfo(&CSEInfo);
217 B.setChangeObserver(Observer);
218
219 RAIIDelegateInstaller DelegateInstaller(MF, &Observer);
220 RAIIMFObserverInstaller MFObserverInstaller(MF, Observer);
221
222 IntrinsicLaneMaskAnalyzer ILMA(MF);
223 MachineRegisterInfo &MRI = *B.getMRI();
224 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
225 RegBankSelectHelper RBSHelper(B, ILMA, MUI, *ST.getRegisterInfo(),
226 *ST.getRegBankInfo());
227 // Virtual registers at this point don't have register banks.
228 // Virtual registers in def and use operands of already inst-selected
229 // instruction have register class.
230
231 for (MachineBasicBlock &MBB : MF) {
232 for (MachineInstr &MI : MBB) {
233 // Vregs in def and use operands of COPY can have either register class
234 // or bank. If there is neither on vreg in def operand, assign bank.
235 if (MI.isCopy()) {
236 Register DefReg = getVReg(Op&: MI.getOperand(i: 0));
237 if (!DefReg.isValid() || MRI.getRegClassOrNull(Reg: DefReg))
238 continue;
239
240 assert(!MRI.getRegBankOrNull(DefReg));
241 MRI.setRegBank(Reg: DefReg, RegBank: *RBSHelper.getRegBankToAssign(Reg: DefReg));
242 continue;
243 }
244
245 if (!MI.isPreISelOpcode())
246 continue;
247
248 // Vregs in def and use operands of G_ instructions need to have register
249 // banks assigned. Before this loop possible case are
250 // - (1) vreg without register class or bank in def or use operand
251 // - (2) vreg with register class in def operand
252 // - (3) vreg, defined by G_ instruction, in use operand
253 // - (4) vreg, defined by pre-inst-selected instruction, in use operand
254
255 // First three cases are handled in loop through all def operands of G_
256 // instructions. For case (1) simply setRegBank. Cases (2) and (3) are
257 // handled by reAssignRegBankOnDef.
258 for (MachineOperand &DefOP : MI.defs()) {
259 Register DefReg = getVReg(Op&: DefOP);
260 if (!DefReg.isValid())
261 continue;
262
263 const RegisterBank *RB = RBSHelper.getRegBankToAssign(Reg: DefReg);
264 if (MRI.getRegClassOrNull(Reg: DefReg))
265 RBSHelper.reAssignRegBankOnDef(MI, DefOP, RB);
266 else {
267 assert(!MRI.getRegBankOrNull(DefReg));
268 MRI.setRegBank(Reg: DefReg, RegBank: *RB);
269 }
270 }
271
272 // Register bank select doesn't modify pre-inst-selected instructions.
273 // For case (4) need to insert a copy, handled by constrainRegBankUse.
274 for (MachineOperand &UseOP : MI.uses()) {
275 Register UseReg = getVReg(Op&: UseOP);
276 if (!UseReg.isValid())
277 continue;
278
279 // Skip case (3).
280 if (!MRI.getRegClassOrNull(Reg: UseReg) ||
281 MRI.getVRegDef(Reg: UseReg)->isPreISelOpcode())
282 continue;
283
284 // Use with register class defined by pre-inst-selected instruction.
285 const RegisterBank *RB = RBSHelper.getRegBankToAssign(Reg: UseReg);
286 RBSHelper.constrainRegBankUse(MI, UseOP, RB);
287 }
288 }
289 }
290
291 return true;
292}
293
294bool AMDGPURegBankSelectLegacy::runOnMachineFunction(MachineFunction &MF) {
295 return runRegBankSelect(
296 MF,
297 GetCSEInfo: [&]() {
298 GISelCSEAnalysisWrapper &Wrapper =
299 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
300 return &Wrapper.get(CSEOpt: getAnalysis<TargetPassConfig>().getCSEConfig());
301 },
302 GetMUI: [&]() {
303 return &getAnalysis<MachineUniformityAnalysisPass>()
304 .getUniformityInfo();
305 });
306}
307
308PreservedAnalyses
309AMDGPURegBankSelectPass::run(MachineFunction &MF,
310 MachineFunctionAnalysisManager &MFAM) {
311 MFPropsModifier _(*this, MF);
312
313 if (!runRegBankSelect(
314 MF, GetCSEInfo: [&]() { return MFAM.getResult<GISelCSEAnalysis>(IR&: MF).get(); },
315 GetMUI: [&]() { return &MFAM.getResult<MachineUniformityAnalysis>(IR&: MF); }))
316 return PreservedAnalyses::all();
317
318 return getMachineFunctionPassPreservedAnalyses();
319}
320