1//===-- GCNPreRAOptimizations.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass combines split register tuple initialization into a single pseudo:
11///
12/// undef %0.sub1:sreg_64 = S_MOV_B32 1
13/// %0.sub0:sreg_64 = S_MOV_B32 2
14/// =>
15/// %0:sreg_64 = S_MOV_B64_IMM_PSEUDO 0x200000001
16///
17/// This is to allow rematerialization of a value instead of spilling. It is
18/// supposed to be done after register coalescer to allow it to do its job and
19/// before actual register allocation to allow rematerialization.
20///
21/// Right now the pass only handles 64 bit SGPRs with immediate initializers,
22/// although the same shall be possible with other register classes and
23/// instructions if necessary.
24///
25/// This pass also adds register allocation hints to COPY.
26/// The hints will be post-processed by SIRegisterInfo::getRegAllocationHints.
27/// When using True16, we often see COPY moving a 16-bit value between a VGPR_32
28/// and a VGPR_16. If we use the VGPR_16 that corresponds to the lo16 bits of
29/// the VGPR_32, the COPY can be completely eliminated.
30///
31//===----------------------------------------------------------------------===//
32
33#include "GCNPreRAOptimizations.h"
34#include "AMDGPU.h"
35#include "GCNSubtarget.h"
36#include "SIRegisterInfo.h"
37#include "llvm/CodeGen/LiveIntervals.h"
38#include "llvm/CodeGen/MachineFunctionPass.h"
39#include "llvm/InitializePasses.h"
40
41using namespace llvm;
42
43#define DEBUG_TYPE "amdgpu-pre-ra-optimizations"
44
45namespace {
46
47class GCNPreRAOptimizationsImpl {
48private:
49 const SIInstrInfo *TII;
50 const SIRegisterInfo *TRI;
51 MachineRegisterInfo *MRI;
52 LiveIntervals *LIS;
53
54 bool processReg(Register Reg);
55 void hintTrue16Copy(const MachineInstr &MI);
56 bool optimizeBVHStack(MachineInstr &MI);
57
58public:
59 GCNPreRAOptimizationsImpl(LiveIntervals *LS) : LIS(LS) {}
60 bool run(MachineFunction &MF);
61};
62
63class GCNPreRAOptimizationsLegacy : public MachineFunctionPass {
64public:
65 static char ID;
66
67 GCNPreRAOptimizationsLegacy() : MachineFunctionPass(ID) {}
68
69 bool runOnMachineFunction(MachineFunction &MF) override;
70
71 StringRef getPassName() const override {
72 return "AMDGPU Pre-RA optimizations";
73 }
74
75 void getAnalysisUsage(AnalysisUsage &AU) const override {
76 AU.addRequired<LiveIntervalsWrapperPass>();
77 AU.setPreservesAll();
78 MachineFunctionPass::getAnalysisUsage(AU);
79 }
80};
81} // End anonymous namespace.
82
83INITIALIZE_PASS_BEGIN(GCNPreRAOptimizationsLegacy, DEBUG_TYPE,
84 "AMDGPU Pre-RA optimizations", false, false)
85INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
86INITIALIZE_PASS_END(GCNPreRAOptimizationsLegacy, DEBUG_TYPE,
87 "Pre-RA optimizations", false, false)
88
89char GCNPreRAOptimizationsLegacy::ID = 0;
90
91char &llvm::GCNPreRAOptimizationsID = GCNPreRAOptimizationsLegacy::ID;
92
93FunctionPass *llvm::createGCNPreRAOptimizationsLegacyPass() {
94 return new GCNPreRAOptimizationsLegacy();
95}
96
97bool GCNPreRAOptimizationsImpl::processReg(Register Reg) {
98 MachineInstr *Def0 = nullptr;
99 MachineInstr *Def1 = nullptr;
100 uint64_t Init = 0;
101 bool Changed = false;
102 SmallSet<Register, 32> ModifiedRegs;
103 bool IsAGPRDst = TRI->isAGPRClass(RC: MRI->getRegClass(Reg));
104
105 for (MachineInstr &I : MRI->def_instructions(Reg)) {
106 switch (I.getOpcode()) {
107 default:
108 return false;
109 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
110 break;
111 case AMDGPU::COPY: {
112 // Some subtargets cannot do an AGPR to AGPR copy directly, and need an
113 // intermdiate temporary VGPR register. Try to find the defining
114 // accvgpr_write to avoid temporary registers.
115
116 if (!IsAGPRDst)
117 return false;
118
119 Register SrcReg = I.getOperand(i: 1).getReg();
120
121 if (!SrcReg.isVirtual())
122 break;
123
124 // Check if source of copy is from another AGPR.
125 bool IsAGPRSrc = TRI->isAGPRClass(RC: MRI->getRegClass(Reg: SrcReg));
126 if (!IsAGPRSrc)
127 break;
128
129 // def_instructions() does not look at subregs so it may give us a
130 // different instruction that defines the same vreg but different subreg
131 // so we have to manually check subreg.
132 Register SrcSubReg = I.getOperand(i: 1).getSubReg();
133 for (auto &Def : MRI->def_instructions(Reg: SrcReg)) {
134 if (SrcSubReg != Def.getOperand(i: 0).getSubReg())
135 continue;
136
137 if (Def.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
138 const MachineOperand &DefSrcMO = Def.getOperand(i: 1);
139
140 // Immediates are not an issue and can be propagated in
141 // postrapseudos pass. Only handle cases where defining
142 // accvgpr_write source is a vreg.
143 if (DefSrcMO.isReg() && DefSrcMO.getReg().isVirtual()) {
144 // Propagate source reg of accvgpr write to this copy instruction
145 I.getOperand(i: 1).setReg(DefSrcMO.getReg());
146 I.getOperand(i: 1).setSubReg(DefSrcMO.getSubReg());
147
148 // Reg uses were changed, collect unique set of registers to update
149 // live intervals at the end.
150 ModifiedRegs.insert(V: DefSrcMO.getReg());
151 ModifiedRegs.insert(V: SrcReg);
152
153 Changed = true;
154 }
155
156 // Found the defining accvgpr_write, stop looking any further.
157 break;
158 }
159 }
160 break;
161 }
162 case AMDGPU::S_MOV_B32:
163 if (I.getOperand(i: 0).getReg() != Reg || !I.getOperand(i: 1).isImm() ||
164 I.getNumOperands() != 2)
165 return false;
166
167 switch (I.getOperand(i: 0).getSubReg()) {
168 default:
169 return false;
170 case AMDGPU::sub0:
171 if (Def0)
172 return false;
173 Def0 = &I;
174 Init |= Lo_32(Value: I.getOperand(i: 1).getImm());
175 break;
176 case AMDGPU::sub1:
177 if (Def1)
178 return false;
179 Def1 = &I;
180 Init |= static_cast<uint64_t>(I.getOperand(i: 1).getImm()) << 32;
181 break;
182 }
183 break;
184 }
185 }
186
187 // For AGPR reg, check if live intervals need to be updated.
188 if (IsAGPRDst) {
189 if (Changed) {
190 for (Register RegToUpdate : ModifiedRegs) {
191 LIS->removeInterval(Reg: RegToUpdate);
192 LIS->createAndComputeVirtRegInterval(Reg: RegToUpdate);
193 }
194 }
195
196 return Changed;
197 }
198
199 // For SGPR reg, check if we can combine instructions.
200 if (!Def0 || !Def1 || Def0->getParent() != Def1->getParent())
201 return Changed;
202
203 LLVM_DEBUG(dbgs() << "Combining:\n " << *Def0 << " " << *Def1
204 << " =>\n");
205
206 if (SlotIndex::isEarlierInstr(A: LIS->getInstructionIndex(Instr: *Def1),
207 B: LIS->getInstructionIndex(Instr: *Def0)))
208 std::swap(a&: Def0, b&: Def1);
209
210 LIS->RemoveMachineInstrFromMaps(MI&: *Def0);
211 LIS->RemoveMachineInstrFromMaps(MI&: *Def1);
212 auto NewI = BuildMI(BB&: *Def0->getParent(), I&: *Def0, MIMD: Def0->getDebugLoc(),
213 MCID: TII->get(Opcode: AMDGPU::S_MOV_B64_IMM_PSEUDO), DestReg: Reg)
214 .addImm(Val: Init);
215
216 Def0->eraseFromParent();
217 Def1->eraseFromParent();
218 LIS->InsertMachineInstrInMaps(MI&: *NewI);
219 LIS->removeInterval(Reg);
220 LIS->createAndComputeVirtRegInterval(Reg);
221
222 LLVM_DEBUG(dbgs() << " " << *NewI);
223
224 return true;
225}
226
227bool GCNPreRAOptimizationsLegacy::runOnMachineFunction(MachineFunction &MF) {
228 if (skipFunction(F: MF.getFunction()))
229 return false;
230 LiveIntervals *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
231 return GCNPreRAOptimizationsImpl(LIS).run(MF);
232}
233
234PreservedAnalyses
235GCNPreRAOptimizationsPass::run(MachineFunction &MF,
236 MachineFunctionAnalysisManager &MFAM) {
237 LiveIntervals *LIS = &MFAM.getResult<LiveIntervalsAnalysis>(IR&: MF);
238 GCNPreRAOptimizationsImpl(LIS).run(MF);
239 return PreservedAnalyses::all();
240}
241
242void GCNPreRAOptimizationsImpl::hintTrue16Copy(const MachineInstr &MI) {
243 Register Dst = MI.getOperand(i: 0).getReg();
244 Register Src = MI.getOperand(i: 1).getReg();
245 const TargetRegisterClass *DstRC = TRI->getRegClassForReg(MRI: *MRI, Reg: Dst);
246 bool IsDst16Bit = AMDGPU::VGPR_16RegClass.hasSubClassEq(RC: DstRC);
247 if (Dst.isVirtual() && IsDst16Bit && Src.isPhysical() &&
248 TRI->getRegClassForReg(MRI: *MRI, Reg: Src) == &AMDGPU::VGPR_32RegClass)
249 MRI->setRegAllocationHint(VReg: Dst, Type: 0, PrefReg: TRI->getSubReg(Reg: Src, Idx: AMDGPU::lo16));
250 if (Src.isVirtual() && MRI->getRegClass(Reg: Src) == &AMDGPU::VGPR_16RegClass &&
251 Dst.isPhysical() && DstRC == &AMDGPU::VGPR_32RegClass)
252 MRI->setRegAllocationHint(VReg: Src, Type: 0, PrefReg: TRI->getSubReg(Reg: Dst, Idx: AMDGPU::lo16));
253 if (!Dst.isVirtual() || !Src.isVirtual())
254 return;
255 if (MRI->getRegClass(Reg: Dst) == &AMDGPU::VGPR_32RegClass &&
256 MRI->getRegClass(Reg: Src) == &AMDGPU::VGPR_16RegClass) {
257 MRI->setRegAllocationHint(VReg: Dst, Type: AMDGPURI::Size32, PrefReg: Src);
258 MRI->setRegAllocationHint(VReg: Src, Type: AMDGPURI::Size16, PrefReg: Dst);
259 }
260 if (IsDst16Bit && MRI->getRegClass(Reg: Src) == &AMDGPU::VGPR_32RegClass)
261 MRI->setRegAllocationHint(VReg: Dst, Type: AMDGPURI::Size16, PrefReg: Src);
262}
263
264bool GCNPreRAOptimizationsImpl::optimizeBVHStack(MachineInstr &MI) {
265 SmallVector<Register, 2> UseRegs;
266
267 // Find BVH sources for this DS_BVH_STACK instruction.
268 auto CheckUse = [&](MachineOperand &Use) {
269 Register Reg = Use.getReg();
270 for (const MachineInstr &Src : MRI->def_instructions(Reg)) {
271 if (!SIInstrInfo::isImage(MI: Src))
272 continue;
273 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: Src.getOpcode());
274 const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
275 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
276 if (!BaseInfo->BVH)
277 continue;
278 UseRegs.push_back(Elt: Reg);
279 break;
280 }
281 };
282 CheckUse(*TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::data0));
283 CheckUse(*TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::data1));
284
285 if (UseRegs.empty())
286 return false;
287
288 // Add implicit uses for entire BVH source registers.
289 // This avoids partial reallocation of register which could
290 // introduce a premature s_wait_bvhcnt.
291 for (Register Reg : UseRegs) {
292 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: false, isImp: true));
293 LIS->removeInterval(Reg);
294 LIS->createAndComputeVirtRegInterval(Reg);
295 }
296 LLVM_DEBUG(dbgs() << "Added implicit uses to: " << MI);
297
298 return true;
299}
300
301bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
302 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
303 TII = ST.getInstrInfo();
304 MRI = &MF.getRegInfo();
305 TRI = ST.getRegisterInfo();
306
307 bool Changed = false;
308
309 for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
310 Register Reg = Register::index2VirtReg(Index: I);
311 if (!LIS->hasInterval(Reg))
312 continue;
313 const TargetRegisterClass *RC = MRI->getRegClass(Reg);
314 if ((RC->getSizeInBits() != 64 || !TRI->isSGPRClass(RC)) &&
315 (ST.hasGFX90AInsts() || !TRI->isAGPRClass(RC)))
316 continue;
317
318 Changed |= processReg(Reg);
319 }
320
321 const bool HasBVHStack = ST.hasBVHDualAndBVH8Insts();
322 const bool HasRealTrue16 = ST.useRealTrue16Insts();
323
324 if (!HasRealTrue16 && !HasBVHStack)
325 return Changed;
326
327 for (MachineBasicBlock &MBB : MF) {
328 for (MachineInstr &MI : MBB) {
329 // Add RA hints to improve True16 COPY elimination.
330 if (HasRealTrue16 && MI.getOpcode() == AMDGPU::COPY) {
331 hintTrue16Copy(MI);
332 continue;
333 }
334 // Add implicit uses to avoid early wait on intersect ray instructions.
335 if (HasBVHStack &&
336 (MI.getOpcode() == AMDGPU::DS_BVH_STACK_RTN_B32 ||
337 MI.getOpcode() == AMDGPU::DS_BVH_STACK_PUSH8_POP1_RTN_B32 ||
338 MI.getOpcode() == AMDGPU::DS_BVH_STACK_PUSH8_POP2_RTN_B64)) {
339 Changed |= optimizeBVHStack(MI);
340 continue;
341 }
342 }
343 }
344
345 return Changed;
346}
347