1//===- SIPreAllocateWWMRegs.cpp - WWM Register Pre-allocation -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Pass to pre-allocated WWM registers
11//
12//===----------------------------------------------------------------------===//
13
14#include "SIPreAllocateWWMRegs.h"
15#include "AMDGPU.h"
16#include "GCNSubtarget.h"
17#include "SIMachineFunctionInfo.h"
18#include "llvm/ADT/PostOrderIterator.h"
19#include "llvm/CodeGen/LiveIntervals.h"
20#include "llvm/CodeGen/LiveRegMatrix.h"
21#include "llvm/CodeGen/MachineFrameInfo.h"
22#include "llvm/CodeGen/MachineFunctionPass.h"
23#include "llvm/CodeGen/RegisterClassInfo.h"
24#include "llvm/CodeGen/VirtRegMap.h"
25#include "llvm/InitializePasses.h"
26
27using namespace llvm;
28
29#define DEBUG_TYPE "si-pre-allocate-wwm-regs"
30
31bool llvm::isPreallocateSGPRSpillVGPRsEnabled(const MachineFunction &MF) {
32 return MF.getSubtarget<GCNSubtarget>()
33 .getCLOpts()
34 .prealloc_sgpr_spill_vgprs ||
35 MF.getFunction().hasFnAttribute(Kind: "amdgpu-prealloc-sgpr-spill-vgprs");
36}
37
38namespace {
39
40class SIPreAllocateWWMRegs {
41private:
42 const SIInstrInfo *TII;
43 const SIRegisterInfo *TRI;
44 MachineRegisterInfo *MRI;
45 LiveIntervals *LIS;
46 LiveRegMatrix *Matrix;
47 VirtRegMap *VRM;
48 RegisterClassInfo &RCI;
49
50 std::vector<unsigned> RegsToRewrite;
51#ifndef NDEBUG
52 void printWWMInfo(const MachineInstr &MI);
53#endif
54 bool processDef(MachineOperand &MO);
55 void rewriteRegs(MachineFunction &MF);
56
57public:
58 SIPreAllocateWWMRegs(LiveIntervals *LIS, LiveRegMatrix *Matrix,
59 VirtRegMap *VRM, RegisterClassInfo &RCI)
60 : LIS(LIS), Matrix(Matrix), VRM(VRM), RCI(RCI) {}
61 bool run(MachineFunction &MF);
62};
63
64class SIPreAllocateWWMRegsLegacy : public MachineFunctionPass {
65public:
66 static char ID;
67
68 SIPreAllocateWWMRegsLegacy() : MachineFunctionPass(ID) {}
69
70 bool runOnMachineFunction(MachineFunction &MF) override;
71
72 void getAnalysisUsage(AnalysisUsage &AU) const override {
73 AU.addRequired<LiveIntervalsWrapperPass>();
74 AU.addRequired<VirtRegMapWrapperLegacy>();
75 AU.addRequired<LiveRegMatrixWrapperLegacy>();
76 AU.addRequired<MachineRegisterClassInfoWrapperPass>();
77 AU.setPreservesAll();
78 MachineFunctionPass::getAnalysisUsage(AU);
79 }
80};
81
82} // End anonymous namespace.
83
84INITIALIZE_PASS_BEGIN(SIPreAllocateWWMRegsLegacy, DEBUG_TYPE,
85 "SI Pre-allocate WWM Registers", false, false)
86INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
87INITIALIZE_PASS_DEPENDENCY(VirtRegMapWrapperLegacy)
88INITIALIZE_PASS_DEPENDENCY(LiveRegMatrixWrapperLegacy)
89INITIALIZE_PASS_DEPENDENCY(MachineRegisterClassInfoWrapperPass)
90INITIALIZE_PASS_END(SIPreAllocateWWMRegsLegacy, DEBUG_TYPE,
91 "SI Pre-allocate WWM Registers", false, false)
92
93char SIPreAllocateWWMRegsLegacy::ID = 0;
94
95char &llvm::SIPreAllocateWWMRegsLegacyID = SIPreAllocateWWMRegsLegacy::ID;
96
97bool SIPreAllocateWWMRegs::processDef(MachineOperand &MO) {
98 Register Reg = MO.getReg();
99 if (Reg.isPhysical())
100 return false;
101
102 if (!SIRegisterInfo::hasVGPRs(RC: MRI->getRegClass(Reg)))
103 return false;
104
105 if (VRM->hasPhys(virtReg: Reg))
106 return false;
107
108 LiveInterval &LI = LIS->getInterval(Reg);
109
110 for (MCRegister PhysReg : RCI.getOrder(RC: MRI->getRegClass(Reg))) {
111 if (!MRI->isPhysRegUsed(PhysReg, /*SkipRegMaskTest=*/true) &&
112 Matrix->checkInterference(VirtReg: LI, PhysReg) == LiveRegMatrix::IK_Free) {
113 Matrix->assign(VirtReg: LI, PhysReg);
114 assert(PhysReg != 0);
115 RegsToRewrite.push_back(x: Reg);
116 return true;
117 }
118 }
119
120 llvm_unreachable("physreg not found for WWM expression");
121}
122
123void SIPreAllocateWWMRegs::rewriteRegs(MachineFunction &MF) {
124 for (MachineBasicBlock &MBB : MF) {
125 for (MachineInstr &MI : MBB) {
126 for (MachineOperand &MO : MI.operands()) {
127 if (!MO.isReg())
128 continue;
129
130 const Register VirtReg = MO.getReg();
131 if (VirtReg.isPhysical())
132 continue;
133
134 if (!VirtReg.isValid())
135 continue;
136
137 if (!VRM->hasPhys(virtReg: VirtReg))
138 continue;
139
140 Register PhysReg = VRM->getPhys(virtReg: VirtReg);
141 const unsigned SubReg = MO.getSubReg();
142 if (SubReg != 0) {
143 PhysReg = TRI->getSubReg(Reg: PhysReg, Idx: SubReg);
144 MO.setSubReg(0);
145 }
146
147 MO.setReg(PhysReg);
148 MO.setIsRenamable(false);
149 }
150 }
151 }
152
153 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
154
155 for (unsigned Reg : RegsToRewrite) {
156 const Register PhysReg = VRM->getPhys(virtReg: Reg);
157 assert(PhysReg != 0);
158
159 LiveInterval &LI = LIS->getInterval(Reg);
160 Matrix->unassign(VirtReg: LI, /*ClearAllReferencingSegments=*/true);
161 LIS->removeInterval(Reg);
162
163 MFI->reserveWWMRegister(Reg: PhysReg);
164 }
165
166 RegsToRewrite.clear();
167
168 // Update the set of reserved registers to include WWM ones
169 // without unnecessarily invalidating RegClassInfo.
170 MRI->freezeReservedRegs();
171 RCI.updateReservedRegs(ReservedInput: MRI->getReservedRegs());
172}
173
174#ifndef NDEBUG
175LLVM_DUMP_METHOD void
176SIPreAllocateWWMRegs::printWWMInfo(const MachineInstr &MI) {
177
178 unsigned Opc = MI.getOpcode();
179
180 if (Opc == AMDGPU::ENTER_STRICT_WWM || Opc == AMDGPU::ENTER_STRICT_WQM) {
181 dbgs() << "Entering ";
182 } else {
183 assert(Opc == AMDGPU::EXIT_STRICT_WWM || Opc == AMDGPU::EXIT_STRICT_WQM);
184 dbgs() << "Exiting ";
185 }
186
187 if (Opc == AMDGPU::ENTER_STRICT_WWM || Opc == AMDGPU::EXIT_STRICT_WWM) {
188 dbgs() << "Strict WWM ";
189 } else {
190 assert(Opc == AMDGPU::ENTER_STRICT_WQM || Opc == AMDGPU::EXIT_STRICT_WQM);
191 dbgs() << "Strict WQM ";
192 }
193
194 dbgs() << "region: " << MI;
195}
196
197#endif
198
199bool SIPreAllocateWWMRegsLegacy::runOnMachineFunction(MachineFunction &MF) {
200 auto *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
201 auto *Matrix = &getAnalysis<LiveRegMatrixWrapperLegacy>().getLRM();
202 auto *VRM = &getAnalysis<VirtRegMapWrapperLegacy>().getVRM();
203 auto &RCI = getAnalysis<MachineRegisterClassInfoWrapperPass>().getRCI();
204 return SIPreAllocateWWMRegs(LIS, Matrix, VRM, RCI).run(MF);
205}
206
207bool SIPreAllocateWWMRegs::run(MachineFunction &MF) {
208 LLVM_DEBUG(dbgs() << "SIPreAllocateWWMRegs: function " << MF.getName() << "\n");
209
210 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
211
212 TII = ST.getInstrInfo();
213 TRI = &TII->getRegisterInfo();
214 MRI = &MF.getRegInfo();
215
216 bool PreallocateSGPRSpillVGPRs = isPreallocateSGPRSpillVGPRsEnabled(MF);
217
218 bool RegsAssigned = false;
219
220 // We use a reverse post-order traversal of the control-flow graph to
221 // guarantee that we visit definitions in dominance order. Since WWM
222 // expressions are guaranteed to never involve phi nodes, and we can only
223 // escape WWM through the special WWM instruction, this means that this is a
224 // perfect elimination order, so we can never do any better.
225 ReversePostOrderTraversal<MachineFunction*> RPOT(&MF);
226
227 for (MachineBasicBlock *MBB : RPOT) {
228 bool InWWM = false;
229 for (MachineInstr &MI : *MBB) {
230 if (MI.getOpcode() == AMDGPU::SI_SPILL_S32_TO_VGPR) {
231 if (PreallocateSGPRSpillVGPRs)
232 RegsAssigned |= processDef(MO&: MI.getOperand(i: 0));
233 continue;
234 }
235
236 if (MI.getOpcode() == AMDGPU::ENTER_STRICT_WWM ||
237 MI.getOpcode() == AMDGPU::ENTER_STRICT_WQM) {
238 LLVM_DEBUG(printWWMInfo(MI));
239 InWWM = true;
240 continue;
241 }
242
243 if (MI.getOpcode() == AMDGPU::EXIT_STRICT_WWM ||
244 MI.getOpcode() == AMDGPU::EXIT_STRICT_WQM) {
245 LLVM_DEBUG(printWWMInfo(MI));
246 InWWM = false;
247 }
248
249 if (!InWWM)
250 continue;
251
252 LLVM_DEBUG(dbgs() << "Processing " << MI);
253
254 for (MachineOperand &DefOpnd : MI.defs()) {
255 RegsAssigned |= processDef(MO&: DefOpnd);
256 }
257 }
258 }
259
260 if (!RegsAssigned)
261 return false;
262
263 rewriteRegs(MF);
264 return true;
265}
266
267PreservedAnalyses
268SIPreAllocateWWMRegsPass::run(MachineFunction &MF,
269 MachineFunctionAnalysisManager &MFAM) {
270 auto *LIS = &MFAM.getResult<LiveIntervalsAnalysis>(IR&: MF);
271 auto *Matrix = &MFAM.getResult<LiveRegMatrixAnalysis>(IR&: MF);
272 auto *VRM = &MFAM.getResult<VirtRegMapAnalysis>(IR&: MF);
273 auto &RCI = MFAM.getResult<MachineRegisterClassAnalysis>(IR&: MF);
274 SIPreAllocateWWMRegs(LIS, Matrix, VRM, RCI).run(MF);
275 return PreservedAnalyses::all();
276}
277