1//===-- GCNPreRAOptimizations.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass combines split register tuple initialization into a single pseudo:
11///
12/// undef %0.sub1:sreg_64 = S_MOV_B32 1
13/// %0.sub0:sreg_64 = S_MOV_B32 2
14/// =>
15/// %0:sreg_64 = S_MOV_B64_IMM_PSEUDO 0x200000001
16///
17/// This is to allow rematerialization of a value instead of spilling. It is
18/// supposed to be done after register coalescer to allow it to do its job and
19/// before actual register allocation to allow rematerialization.
20///
21/// Right now the pass only handles 64 bit SGPRs with immediate initializers,
22/// although the same shall be possible with other register classes and
23/// instructions if necessary.
24///
25/// This pass also adds register allocation hints to COPY.
26/// The hints will be post-processed by SIRegisterInfo::getRegAllocationHints.
27/// When using True16, we often see COPY moving a 16-bit value between a VGPR_32
28/// and a VGPR_16. If we use the VGPR_16 that corresponds to the lo16 bits of
29/// the VGPR_32, the COPY can be completely eliminated.
30///
31//===----------------------------------------------------------------------===//
32
33#include "GCNPreRAOptimizations.h"
34#include "AMDGPU.h"
35#include "GCNPreRAAntiHints.h"
36#include "GCNSubtarget.h"
37#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
38#include "SIInstrInfo.h"
39#include "SIRegisterInfo.h"
40#include "llvm/CodeGen/LiveIntervals.h"
41#include "llvm/CodeGen/MachineFunctionPass.h"
42#include "llvm/CodeGen/Register.h"
43#include "llvm/CodeGen/TargetSchedule.h"
44#include "llvm/InitializePasses.h"
45
46using namespace llvm;
47
48#define DEBUG_TYPE "amdgpu-pre-ra-optimizations"
49
50namespace {
51
52class GCNPreRAOptimizationsImpl {
53private:
54 const SIInstrInfo *TII;
55 const SIRegisterInfo *TRI;
56 MachineRegisterInfo *MRI;
57 LiveIntervals *LIS;
58 TargetSchedModel SchedModel;
59
60 bool processReg(Register Reg);
61 void hintTrue16Copy(const MachineInstr &MI);
62 bool optimizeBVHStack(MachineInstr &MI);
63
64public:
65 GCNPreRAOptimizationsImpl(LiveIntervals *LS) : LIS(LS) {}
66 bool run(MachineFunction &MF);
67};
68
69class GCNPreRAOptimizationsLegacy : public MachineFunctionPass {
70public:
71 static char ID;
72
73 GCNPreRAOptimizationsLegacy() : MachineFunctionPass(ID) {}
74
75 bool runOnMachineFunction(MachineFunction &MF) override;
76
77 StringRef getPassName() const override {
78 return "AMDGPU Pre-RA optimizations";
79 }
80
81 void getAnalysisUsage(AnalysisUsage &AU) const override {
82 AU.addRequired<LiveIntervalsWrapperPass>();
83 AU.setPreservesAll();
84 MachineFunctionPass::getAnalysisUsage(AU);
85 }
86};
87} // End anonymous namespace.
88
89INITIALIZE_PASS_BEGIN(GCNPreRAOptimizationsLegacy, DEBUG_TYPE,
90 "AMDGPU Pre-RA optimizations", false, false)
91INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
92INITIALIZE_PASS_END(GCNPreRAOptimizationsLegacy, DEBUG_TYPE,
93 "Pre-RA optimizations", false, false)
94
95char GCNPreRAOptimizationsLegacy::ID = 0;
96
97char &llvm::GCNPreRAOptimizationsID = GCNPreRAOptimizationsLegacy::ID;
98
99bool GCNPreRAOptimizationsImpl::processReg(Register Reg) {
100 MachineInstr *Def0 = nullptr;
101 MachineInstr *Def1 = nullptr;
102 uint64_t Init = 0;
103 bool Changed = false;
104 SmallSet<Register, 32> ModifiedRegs;
105 bool IsAGPRDst = TRI->isAGPRClass(RC: MRI->getRegClass(Reg));
106
107 for (MachineInstr &I : MRI->def_instructions(Reg)) {
108 switch (I.getOpcode()) {
109 default:
110 return false;
111 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
112 break;
113 case AMDGPU::COPY: {
114 // Some subtargets cannot do an AGPR to AGPR copy directly, and need an
115 // intermdiate temporary VGPR register. Try to find the defining
116 // accvgpr_write to avoid temporary registers.
117
118 if (!IsAGPRDst)
119 return false;
120
121 Register SrcReg = I.getOperand(i: 1).getReg();
122
123 if (!SrcReg.isVirtual())
124 break;
125
126 // Check if source of copy is from another AGPR.
127 bool IsAGPRSrc = TRI->isAGPRClass(RC: MRI->getRegClass(Reg: SrcReg));
128 if (!IsAGPRSrc)
129 break;
130
131 // def_instructions() does not look at subregs so it may give us a
132 // different instruction that defines the same vreg but different subreg
133 // so we have to manually check subreg.
134 Register SrcSubReg = I.getOperand(i: 1).getSubReg();
135 for (auto &Def : MRI->def_instructions(Reg: SrcReg)) {
136 if (SrcSubReg != Def.getOperand(i: 0).getSubReg())
137 continue;
138
139 if (Def.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
140 const MachineOperand &DefSrcMO = Def.getOperand(i: 1);
141
142 // Immediates are not an issue and can be propagated in
143 // postrapseudos pass. Only handle cases where defining
144 // accvgpr_write source is a vreg.
145 if (DefSrcMO.isReg() && DefSrcMO.getReg().isVirtual()) {
146 // Propagate source reg of accvgpr write to this copy instruction
147 I.getOperand(i: 1).setReg(DefSrcMO.getReg());
148 I.getOperand(i: 1).setSubReg(DefSrcMO.getSubReg());
149
150 // Reg uses were changed, collect unique set of registers to update
151 // live intervals at the end.
152 ModifiedRegs.insert(V: DefSrcMO.getReg());
153 ModifiedRegs.insert(V: SrcReg);
154
155 Changed = true;
156 }
157
158 // Found the defining accvgpr_write, stop looking any further.
159 break;
160 }
161 }
162 break;
163 }
164 case AMDGPU::S_MOV_B32:
165 if (I.getOperand(i: 0).getReg() != Reg || !I.getOperand(i: 1).isImm() ||
166 I.getNumOperands() != 2)
167 return false;
168
169 switch (I.getOperand(i: 0).getSubReg()) {
170 default:
171 return false;
172 case AMDGPU::sub0:
173 if (Def0)
174 return false;
175 Def0 = &I;
176 Init |= Lo_32(Value: I.getOperand(i: 1).getImm());
177 break;
178 case AMDGPU::sub1:
179 if (Def1)
180 return false;
181 Def1 = &I;
182 Init |= static_cast<uint64_t>(I.getOperand(i: 1).getImm()) << 32;
183 break;
184 }
185 break;
186 }
187 }
188
189 // For AGPR reg, check if live intervals need to be updated.
190 if (IsAGPRDst) {
191 if (Changed) {
192 for (Register RegToUpdate : ModifiedRegs) {
193 LIS->removeInterval(Reg: RegToUpdate);
194 LIS->createAndComputeVirtRegInterval(Reg: RegToUpdate);
195 }
196 }
197
198 return Changed;
199 }
200
201 // For SGPR reg, check if we can combine instructions.
202 if (!Def0 || !Def1 || Def0->getParent() != Def1->getParent())
203 return Changed;
204
205 LLVM_DEBUG(dbgs() << "Combining:\n " << *Def0 << " " << *Def1
206 << " =>\n");
207
208 if (SlotIndex::isEarlierInstr(A: LIS->getInstructionIndex(Instr: *Def1),
209 B: LIS->getInstructionIndex(Instr: *Def0)))
210 std::swap(a&: Def0, b&: Def1);
211
212 LIS->RemoveMachineInstrFromMaps(MI&: *Def0);
213 LIS->RemoveMachineInstrFromMaps(MI&: *Def1);
214 auto NewI = BuildMI(BB&: *Def0->getParent(), I&: *Def0, MIMD: Def0->getDebugLoc(),
215 MCID: TII->get(Opcode: AMDGPU::S_MOV_B64_IMM_PSEUDO), DestReg: Reg)
216 .addImm(Val: Init);
217
218 Def0->eraseFromParent();
219 Def1->eraseFromParent();
220 LIS->InsertMachineInstrInMaps(MI&: *NewI);
221 LIS->removeInterval(Reg);
222 LIS->createAndComputeVirtRegInterval(Reg);
223
224 LLVM_DEBUG(dbgs() << " " << *NewI);
225
226 return true;
227}
228
229bool GCNPreRAOptimizationsLegacy::runOnMachineFunction(MachineFunction &MF) {
230 if (skipFunction(F: MF.getFunction()))
231 return false;
232 LiveIntervals *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
233 return GCNPreRAOptimizationsImpl(LIS).run(MF);
234}
235
236PreservedAnalyses
237GCNPreRAOptimizationsPass::run(MachineFunction &MF,
238 MachineFunctionAnalysisManager &MFAM) {
239 LiveIntervals *LIS = &MFAM.getResult<LiveIntervalsAnalysis>(IR&: MF);
240 GCNPreRAOptimizationsImpl(LIS).run(MF);
241 return PreservedAnalyses::all();
242}
243
244void GCNPreRAOptimizationsImpl::hintTrue16Copy(const MachineInstr &MI) {
245 Register Dst = MI.getOperand(i: 0).getReg();
246 Register Src = MI.getOperand(i: 1).getReg();
247 const TargetRegisterClass *DstRC = TRI->getRegClassForReg(MRI: *MRI, Reg: Dst);
248 bool IsDst16Bit = AMDGPU::VGPR_16RegClass.hasSubClassEq(RC: DstRC);
249 if (Dst.isVirtual() && IsDst16Bit && Src.isPhysical() &&
250 TRI->getRegClassForReg(MRI: *MRI, Reg: Src) == &AMDGPU::VGPR_32RegClass)
251 MRI->setRegAllocationHint(VReg: Dst, Type: 0, PrefReg: TRI->getSubReg(Reg: Src, Idx: AMDGPU::lo16));
252 if (Src.isVirtual() && MRI->getRegClass(Reg: Src) == &AMDGPU::VGPR_16RegClass &&
253 Dst.isPhysical() && DstRC == &AMDGPU::VGPR_32RegClass)
254 MRI->setRegAllocationHint(VReg: Src, Type: 0, PrefReg: TRI->getSubReg(Reg: Dst, Idx: AMDGPU::lo16));
255 if (!Dst.isVirtual() || !Src.isVirtual())
256 return;
257 if (MRI->getRegClass(Reg: Dst) == &AMDGPU::VGPR_32RegClass &&
258 MRI->getRegClass(Reg: Src) == &AMDGPU::VGPR_16RegClass) {
259 MRI->setRegAllocationHint(VReg: Dst, Type: AMDGPURI::Size32, PrefReg: Src);
260 MRI->setRegAllocationHint(VReg: Src, Type: AMDGPURI::Size16, PrefReg: Dst);
261 }
262 if (IsDst16Bit && MRI->getRegClass(Reg: Src) == &AMDGPU::VGPR_32RegClass)
263 MRI->setRegAllocationHint(VReg: Dst, Type: AMDGPURI::Size16, PrefReg: Src);
264}
265
266bool GCNPreRAOptimizationsImpl::optimizeBVHStack(MachineInstr &MI) {
267 SmallVector<Register, 2> UseRegs;
268
269 // Find BVH sources for this DS_BVH_STACK instruction.
270 auto CheckUse = [&](MachineOperand &Use) {
271 Register Reg = Use.getReg();
272 for (const MachineInstr &Src : MRI->def_instructions(Reg)) {
273 if (!SIInstrInfo::isImage(MI: Src))
274 continue;
275 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: Src.getOpcode());
276 const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
277 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
278 if (!BaseInfo->BVH)
279 continue;
280 UseRegs.push_back(Elt: Reg);
281 break;
282 }
283 };
284 CheckUse(*TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::data0));
285 CheckUse(*TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::data1));
286
287 if (UseRegs.empty())
288 return false;
289
290 // Add implicit uses for entire BVH source registers.
291 // This avoids partial reallocation of register which could
292 // introduce a premature s_wait_bvhcnt.
293 for (Register Reg : UseRegs) {
294 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: false, isImp: true));
295 LIS->removeInterval(Reg);
296 LIS->createAndComputeVirtRegInterval(Reg);
297 }
298 LLVM_DEBUG(dbgs() << "Added implicit uses to: " << MI);
299
300 return true;
301}
302
303bool GCNPreRAOptimizationsImpl::run(MachineFunction &MF) {
304 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
305 TII = ST.getInstrInfo();
306 MRI = &MF.getRegInfo();
307 TRI = ST.getRegisterInfo();
308
309 // Anti-hints only steer register allocation, so they do not count as a
310 // modification of the function.
311 if (ST.getCLOpts().anti_hints) {
312 SchedModel.init(TSInfo: &ST);
313 AMDGPU::HazardContext HCtx{.TII: TII, .TRI: TRI, .MRI: MRI, .LIS: LIS, .ST: &ST, .SchedModel: &SchedModel};
314 AMDGPU::applyAntiHintRules(MF, Ctx: HCtx);
315 }
316
317 bool Changed = false;
318
319 for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
320 Register Reg = Register::index2VirtReg(Index: I);
321 if (!LIS->hasInterval(Reg))
322 continue;
323 const TargetRegisterClass *RC = MRI->getRegClass(Reg);
324 if ((RC->getSizeInBits() != 64 || !TRI->isSGPRClass(RC)) &&
325 (ST.hasGFX90AInsts() || !TRI->isAGPRClass(RC)))
326 continue;
327
328 Changed |= processReg(Reg);
329 }
330
331 const bool HasBVHStack = ST.hasBVHDualAndBVH8Insts();
332 const bool HasRealTrue16 = ST.useRealTrue16Insts();
333
334 if (!HasRealTrue16 && !HasBVHStack)
335 return Changed;
336
337 for (MachineBasicBlock &MBB : MF) {
338 for (MachineInstr &MI : MBB) {
339 // Add RA hints to improve True16 COPY elimination.
340 if (HasRealTrue16 && MI.getOpcode() == AMDGPU::COPY) {
341 hintTrue16Copy(MI);
342 continue;
343 }
344 // Add implicit uses to avoid early wait on intersect ray instructions.
345 if (HasBVHStack &&
346 (MI.getOpcode() == AMDGPU::DS_BVH_STACK_RTN_B32 ||
347 MI.getOpcode() == AMDGPU::DS_BVH_STACK_PUSH8_POP1_RTN_B32 ||
348 MI.getOpcode() == AMDGPU::DS_BVH_STACK_PUSH8_POP2_RTN_B64)) {
349 Changed |= optimizeBVHStack(MI);
350 continue;
351 }
352 }
353 }
354
355 return Changed;
356}
357