1//===- AMDGPUWaitSGPRHazards.cpp - Insert waits for SGPR read hazards -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Insert s_wait_alu instructions to mitigate SGPR read hazards on GFX12.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUWaitSGPRHazards.h"
15#include "AMDGPU.h"
16#include "GCNSubtarget.h"
17#include "SIInstrInfo.h"
18#include "llvm/ADT/SmallBitVector.h"
19#include "llvm/TargetParser/AMDGPUTargetParser.h"
20
21using namespace llvm;
22
23#define DEBUG_TYPE "amdgpu-wait-sgpr-hazards"
24
25static cl::opt<bool> GlobalCullSGPRHazardsOnFunctionBoundary(
26 "amdgpu-sgpr-hazard-boundary-cull", cl::init(Val: false), cl::Hidden,
27 cl::desc("Cull hazards on function boundaries"));
28
29static cl::opt<bool>
30 GlobalCullSGPRHazardsAtMemWait("amdgpu-sgpr-hazard-mem-wait-cull",
31 cl::init(Val: false), cl::Hidden,
32 cl::desc("Cull hazards on memory waits"));
33
34static cl::opt<unsigned> GlobalCullSGPRHazardsMemWaitThreshold(
35 "amdgpu-sgpr-hazard-mem-wait-cull-threshold", cl::init(Val: 8), cl::Hidden,
36 cl::desc("Number of tracked SGPRs before initiating hazard cull on memory "
37 "wait"));
38
39namespace {
40
41class AMDGPUWaitSGPRHazards {
42public:
43 const GCNSubtarget *ST;
44 const SIInstrInfo *TII;
45 const SIRegisterInfo *TRI;
46 const MachineRegisterInfo *MRI;
47 unsigned DsNopCount;
48
49 bool CullSGPRHazardsOnFunctionBoundary;
50 bool CullSGPRHazardsAtMemWait;
51 unsigned CullSGPRHazardsMemWaitThreshold;
52
53 AMDGPUWaitSGPRHazards() = default;
54
55 // Return the numeric ID 0-127 for a given SGPR.
56 static std::optional<unsigned> sgprNumber(Register Reg,
57 const SIRegisterInfo &TRI) {
58 switch (Reg) {
59 case AMDGPU::M0:
60 case AMDGPU::EXEC:
61 case AMDGPU::EXEC_LO:
62 case AMDGPU::EXEC_HI:
63 case AMDGPU::SGPR_NULL:
64 case AMDGPU::SGPR_NULL64:
65 case AMDGPU::SCC:
66 return {};
67 default:
68 break;
69 }
70 unsigned RegN = TRI.getHWRegIndex(Reg);
71 if (RegN > 127)
72 return {};
73 return RegN;
74 }
75
76 static inline bool isVCC(Register Reg) {
77 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI;
78 }
79
80 // Adjust global offsets for instructions bundled with S_GETPC_B64 after
81 // insertion of a new instruction.
82 static void updateGetPCBundle(MachineInstr *NewMI) {
83 if (!NewMI->isBundled())
84 return;
85
86 // Find start of bundle.
87 auto I = NewMI->getIterator();
88 while (I->isBundledWithPred())
89 I--;
90 if (I->isBundle())
91 I++;
92
93 // Bail if this is not an S_GETPC bundle.
94 if (I->getOpcode() != AMDGPU::S_GETPC_B64)
95 return;
96
97 // Update offsets of any references in the bundle.
98 const unsigned NewBytes = 4;
99 assert(NewMI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
100 "Unexpected instruction insertion in bundle");
101 auto NextMI = std::next(x: NewMI->getIterator());
102 auto End = NewMI->getParent()->end();
103 while (NextMI != End && NextMI->isBundledWithPred()) {
104 for (auto &Operand : NextMI->operands()) {
105 if (Operand.isGlobal())
106 Operand.setOffset(Operand.getOffset() + NewBytes);
107 }
108 NextMI++;
109 }
110 }
111
112 struct HazardState {
113 static constexpr unsigned None = 0;
114 static constexpr unsigned SALU = (1 << 0);
115 static constexpr unsigned VALU = (1 << 1);
116
117 std::bitset<64> Tracked; // SGPR banks ever read by VALU
118 std::bitset<128> SALUHazards; // SGPRs with uncommitted values from SALU
119 std::bitset<128> VALUHazards; // SGPRs with uncommitted values from VALU
120 unsigned VCCHazard = None; // Source of current VCC writes
121 bool ActiveFlat = false; // Has unwaited flat instructions
122
123 bool merge(const HazardState &RHS) {
124 HazardState Orig(*this);
125 *this |= RHS;
126 return (*this != Orig);
127 }
128
129 bool operator==(const HazardState &RHS) const {
130 return Tracked == RHS.Tracked && SALUHazards == RHS.SALUHazards &&
131 VALUHazards == RHS.VALUHazards && VCCHazard == RHS.VCCHazard &&
132 ActiveFlat == RHS.ActiveFlat;
133 }
134
135 bool operator!=(const HazardState &RHS) const { return !(*this == RHS); }
136
137 void operator|=(const HazardState &RHS) {
138 Tracked |= RHS.Tracked;
139 SALUHazards |= RHS.SALUHazards;
140 VALUHazards |= RHS.VALUHazards;
141 VCCHazard |= RHS.VCCHazard;
142 ActiveFlat |= RHS.ActiveFlat;
143 }
144 };
145
146 struct BlockHazardState {
147 HazardState In;
148 HazardState Out;
149 };
150
151 DenseMap<const MachineBasicBlock *, BlockHazardState> BlockState;
152
153 static constexpr unsigned WAVE32_NOPS = 4;
154 static constexpr unsigned WAVE64_NOPS = 8;
155
156 void insertHazardCull(MachineBasicBlock &MBB,
157 MachineBasicBlock::instr_iterator &MI) {
158 assert(!MI->isBundled());
159 unsigned Count = DsNopCount;
160 while (Count--)
161 BuildMI(BB&: MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::DS_NOP));
162 }
163
164 unsigned mergeMasks(unsigned Mask1, unsigned Mask2) {
165 unsigned Mask = AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: *ST);
166 Mask = AMDGPU::DepCtr::encodeFieldSaSdst(
167 Encoded: Mask, SaSdst: std::min(a: AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Mask1),
168 b: AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Mask2)));
169 Mask = AMDGPU::DepCtr::encodeFieldVaVcc(
170 Encoded: Mask, VaVcc: std::min(a: AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Mask1),
171 b: AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Mask2)));
172 Mask = AMDGPU::DepCtr::encodeFieldVmVsrc(
173 Encoded: Mask, VmVsrc: std::min(a: AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: Mask1),
174 b: AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: Mask2)));
175 Mask = AMDGPU::DepCtr::encodeFieldVaSdst(
176 Encoded: Mask, VaSdst: std::min(a: AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Mask1),
177 b: AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Mask2)));
178 Mask = AMDGPU::DepCtr::encodeFieldVaVdst(
179 Encoded: Mask, VaVdst: std::min(a: AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: Mask1),
180 b: AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: Mask2)));
181 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: ST->getCPU());
182 Mask = AMDGPU::DepCtr::encodeFieldHoldCnt(
183 Encoded: Mask,
184 HoldCnt: std::min(a: AMDGPU::DepCtr::decodeFieldHoldCnt(Encoded: Mask1, Version),
185 b: AMDGPU::DepCtr::decodeFieldHoldCnt(Encoded: Mask2, Version)),
186 Version);
187 Mask = AMDGPU::DepCtr::encodeFieldVaSsrc(
188 Encoded: Mask, VaSsrc: std::min(a: AMDGPU::DepCtr::decodeFieldVaSsrc(Encoded: Mask1),
189 b: AMDGPU::DepCtr::decodeFieldVaSsrc(Encoded: Mask2)));
190 return Mask;
191 }
192
193 bool mergeConsecutiveWaitAlus(MachineBasicBlock::instr_iterator &MI,
194 unsigned Mask) {
195 auto MBB = MI->getParent();
196 if (MI == MBB->instr_begin())
197 return false;
198
199 auto It = prev_nodbg(It: MI, Begin: MBB->instr_begin());
200 if (It->getOpcode() != AMDGPU::S_WAITCNT_DEPCTR)
201 return false;
202
203 It->getOperand(i: 0).setImm(
204 mergeMasks(Mask1: Mask, Mask2: static_cast<unsigned>(It->getOperand(i: 0).getImm())));
205 return true;
206 }
207
208 bool runOnMachineBasicBlock(MachineBasicBlock &MBB, bool Emit) {
209 enum { WA_VALU = 0x1, WA_SALU = 0x2, WA_VCC = 0x4 };
210
211 HazardState State = BlockState[&MBB].In;
212 SmallSet<Register, 8> SeenRegs;
213 bool Emitted = false;
214 unsigned DsNops = 0;
215
216 for (MachineBasicBlock::instr_iterator MI = MBB.instr_begin(),
217 E = MBB.instr_end();
218 MI != E; ++MI) {
219 if (MI->isMetaInstruction())
220 continue;
221
222 // Clear tracked SGPRs if sufficient DS_NOPs occur
223 if (MI->getOpcode() == AMDGPU::DS_NOP) {
224 if (++DsNops >= DsNopCount)
225 State.Tracked.reset();
226 continue;
227 }
228 DsNops = 0;
229
230 // Snoop FLAT instructions to avoid adding culls before scratch/lds loads.
231 // Culls could be disproportionate in cost to load time.
232 if (SIInstrInfo::isFLAT(MI: *MI) && !SIInstrInfo::isFLATGlobal(MI: *MI))
233 State.ActiveFlat = true;
234
235 // SMEM or VMEM clears hazards
236 // FIXME: adapt to add FLAT without VALU (so !isLDSDMA())?
237 if ((SIInstrInfo::isVMEM(MI: *MI) && !SIInstrInfo::isFLAT(MI: *MI)) ||
238 SIInstrInfo::isSMRD(MI: *MI)) {
239 State.VCCHazard = HazardState::None;
240 State.SALUHazards.reset();
241 State.VALUHazards.reset();
242 continue;
243 }
244
245 // Existing S_WAITALU can clear hazards
246 if (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR) {
247 unsigned int Mask = static_cast<unsigned>(MI->getOperand(i: 0).getImm());
248 if (AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Mask) == 0)
249 State.VCCHazard &= ~HazardState::VALU;
250 if (AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Mask) == 0) {
251 State.SALUHazards.reset();
252 State.VCCHazard &= ~HazardState::SALU;
253 }
254 if (AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Mask) == 0)
255 State.VALUHazards.reset();
256 continue;
257 }
258
259 // Snoop counter waits to insert culls
260 if (CullSGPRHazardsAtMemWait &&
261 (MI->getOpcode() == AMDGPU::S_WAIT_LOADCNT ||
262 MI->getOpcode() == AMDGPU::S_WAIT_SAMPLECNT ||
263 MI->getOpcode() == AMDGPU::S_WAIT_BVHCNT) &&
264 (MI->getOperand(i: 0).isImm() && MI->getOperand(i: 0).getImm() == 0) &&
265 (State.Tracked.count() >= CullSGPRHazardsMemWaitThreshold)) {
266 if (MI->getOpcode() == AMDGPU::S_WAIT_LOADCNT && State.ActiveFlat) {
267 State.ActiveFlat = false;
268 } else {
269 State.Tracked.reset();
270 if (Emit)
271 insertHazardCull(MBB, MI);
272 continue;
273 }
274 }
275
276 // Process only VALUs and SALUs
277 bool IsVALU = SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/false);
278 bool IsSALU = SIInstrInfo::isSALU(MI: *MI);
279 if (!IsVALU && !IsSALU)
280 continue;
281
282 unsigned Wait = 0;
283
284 auto processOperand = [&](const MachineOperand &Op, bool IsUse) {
285 if (!Op.isReg())
286 return;
287 Register Reg = Op.getReg();
288 assert(!Op.getSubReg());
289 if (!TRI->isSGPRReg(MRI: *MRI, Reg))
290 return;
291
292 // Only visit each register once
293 if (!SeenRegs.insert(V: Reg).second)
294 return;
295
296 auto RegNumber = sgprNumber(Reg, TRI: *TRI);
297 if (!RegNumber)
298 return;
299
300 // Track SGPRs by pair -- numeric ID of an 64b SGPR pair.
301 // i.e. SGPR0 = SGPR0_SGPR1 = 0, SGPR3 = SGPR2_SGPR3 = 1, etc
302 unsigned RegN = *RegNumber;
303 unsigned PairN = (RegN >> 1) & 0x3f;
304
305 // Read/write of untracked register is safe; but must record any new
306 // reads.
307 if (!State.Tracked[PairN]) {
308 if (IsVALU && IsUse)
309 State.Tracked.set(position: PairN);
310 return;
311 }
312
313 unsigned SGPRCount =
314 AMDGPU::getRegBitWidth(RC: *TRI->getRegClassForReg(MRI: *MRI, Reg)) / 32;
315
316 if (IsUse) {
317 // SALU reading SGPR clears VALU hazards
318 if (IsSALU) {
319 if (isVCC(Reg)) {
320 if (State.VCCHazard & HazardState::VALU)
321 State.VCCHazard = HazardState::None;
322 } else {
323 State.VALUHazards.reset();
324 }
325 }
326 // Compute required waits
327 for (unsigned RegIdx = 0; RegIdx < SGPRCount; ++RegIdx) {
328 Wait |= State.SALUHazards[RegN + RegIdx] ? WA_SALU : 0;
329 Wait |= IsVALU && State.VALUHazards[RegN + RegIdx] ? WA_VALU : 0;
330 }
331 if (isVCC(Reg) && State.VCCHazard) {
332 // Note: it's possible for both SALU and VALU to exist if VCC
333 // was updated differently by merged predecessors.
334 if (State.VCCHazard & HazardState::SALU)
335 Wait |= WA_SALU;
336 if (State.VCCHazard & HazardState::VALU)
337 Wait |= WA_VCC;
338 }
339 } else {
340 // Update hazards
341 if (isVCC(Reg)) {
342 State.VCCHazard = IsSALU ? HazardState::SALU : HazardState::VALU;
343 } else {
344 for (unsigned RegIdx = 0; RegIdx < SGPRCount; ++RegIdx) {
345 if (IsSALU)
346 State.SALUHazards.set(position: RegN + RegIdx);
347 else
348 State.VALUHazards.set(position: RegN + RegIdx);
349 }
350 }
351 }
352 };
353
354 const bool IsSetPC =
355 (MI->isCall() || MI->isReturn() || MI->isIndirectBranch()) &&
356 MI->getOpcode() != AMDGPU::S_ENDPGM &&
357 MI->getOpcode() != AMDGPU::S_ENDPGM_SAVED;
358
359 // Only consider implicit VCC specified by instruction descriptor.
360 const bool HasImplicitVCC =
361 llvm::any_of(Range: MI->getDesc().implicit_uses(), P: isVCC) ||
362 llvm::any_of(Range: MI->getDesc().implicit_defs(), P: isVCC);
363
364 if (IsSetPC) {
365 // All SGPR writes before a call/return must be flushed as the
366 // callee/caller will not will not see the hazard chain.
367 if (State.VCCHazard & HazardState::VALU)
368 Wait |= WA_VCC;
369 if (State.SALUHazards.any() || (State.VCCHazard & HazardState::SALU))
370 Wait |= WA_SALU;
371 if (State.VALUHazards.any())
372 Wait |= WA_VALU;
373 if (CullSGPRHazardsOnFunctionBoundary && State.Tracked.any()) {
374 State.Tracked.reset();
375 if (Emit)
376 insertHazardCull(MBB, MI);
377 }
378 } else {
379 // Process uses to determine required wait.
380 SeenRegs.clear();
381 for (const MachineOperand &Op : MI->all_uses()) {
382 if (Op.isImplicit() &&
383 (!HasImplicitVCC || !Op.isReg() || !isVCC(Reg: Op.getReg())))
384 continue;
385 processOperand(Op, true);
386 }
387 }
388
389 // Apply wait
390 if (Wait) {
391 unsigned Mask = AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: *ST);
392 if (Wait & WA_VCC) {
393 State.VCCHazard &= ~HazardState::VALU;
394 Mask = AMDGPU::DepCtr::encodeFieldVaVcc(Encoded: Mask, VaVcc: 0);
395 }
396 if (Wait & WA_SALU) {
397 State.SALUHazards.reset();
398 State.VCCHazard &= ~HazardState::SALU;
399 Mask = AMDGPU::DepCtr::encodeFieldSaSdst(Encoded: Mask, SaSdst: 0);
400 }
401 if (Wait & WA_VALU) {
402 State.VALUHazards.reset();
403 Mask = AMDGPU::DepCtr::encodeFieldVaSdst(Encoded: Mask, VaSdst: 0);
404 }
405 if (Emit) {
406 if (!mergeConsecutiveWaitAlus(MI, Mask)) {
407 auto NewMI = BuildMI(BB&: MBB, I: MI, MIMD: MI->getDebugLoc(),
408 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
409 .addImm(Val: Mask);
410 updateGetPCBundle(NewMI);
411 }
412 Emitted = true;
413 }
414 }
415
416 // On return from a call SGPR state is unknown, so all potential hazards.
417 if (MI->isCall() && !CullSGPRHazardsOnFunctionBoundary)
418 State.Tracked.set();
419
420 // Update hazards based on defs.
421 SeenRegs.clear();
422 for (const MachineOperand &Op : MI->all_defs()) {
423 if (Op.isImplicit() &&
424 (!HasImplicitVCC || !Op.isReg() || !isVCC(Reg: Op.getReg())))
425 continue;
426 processOperand(Op, false);
427 }
428 }
429
430 BlockHazardState &BS = BlockState[&MBB];
431 bool Changed = State != BS.Out;
432 if (Emit) {
433 assert(!Changed && "Hazard state should not change on emit pass");
434 return Emitted;
435 }
436 if (Changed)
437 BS.Out = State;
438 return Changed;
439 }
440
441 bool runWaitMerging(MachineFunction &MF) {
442 // Perform per-block merging of existing s_waitcnt_depctr instructions.
443 // Track set of SGPR writes before a given wait instruction, and search
444 // for reads of these SGPRs.
445 // Move the wait to just before the read to improve pipelining.
446 // If no related reads occur before subsequent wait then merged waits.
447 const unsigned ConstantMaskBits = AMDGPU::DepCtr::encodeFieldSaSdst(
448 Encoded: AMDGPU::DepCtr::encodeFieldVaSdst(
449 Encoded: AMDGPU::DepCtr::encodeFieldVaVcc(VaVcc: 0, STI: *ST), VaSdst: 0),
450 SaSdst: 0);
451 const unsigned VccLoIdx = *sgprNumber(Reg: AMDGPU::VCC_LO, TRI: *TRI);
452 const unsigned VccHiIdx = *sgprNumber(Reg: AMDGPU::VCC_HI, TRI: *TRI);
453 bool Changed = false;
454 for (MachineBasicBlock &MBB : MF) {
455 SmallBitVector WriteSet(128), PendingSALUWriteSet(128),
456 PendingVALUWriteSet(128);
457 MachineInstr *PrevWait = nullptr;
458
459 auto CommitWrites = [&](unsigned Mask) {
460 if (!AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Mask))
461 WriteSet |= PendingSALUWriteSet;
462 bool VccLoBit = WriteSet[VccLoIdx];
463 bool VccHiBit = WriteSet[VccHiIdx];
464 if (!AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Mask)) {
465 // Apply pending VALU set minus VCC bits
466 WriteSet |= PendingVALUWriteSet;
467 WriteSet[VccLoIdx] = VccLoBit;
468 WriteSet[VccHiIdx] = VccHiBit;
469 }
470 if (!AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Mask)) {
471 WriteSet[VccLoIdx] = VccLoBit || PendingVALUWriteSet[VccLoIdx];
472 WriteSet[VccHiIdx] = VccHiBit || PendingVALUWriteSet[VccHiIdx];
473 }
474 // Clear all pending writes
475 PendingSALUWriteSet.reset();
476 PendingVALUWriteSet.reset();
477 };
478
479 for (MachineInstr &MI : MBB) {
480 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
481 (MI.getOperand(i: 0).getImm() & ConstantMaskBits) ==
482 ConstantMaskBits) {
483 if (PrevWait) {
484 // Merge previous wait into this one.
485 MachineOperand &MaskOp = MI.getOperand(i: 0);
486 MaskOp.setImm(mergeMasks(
487 Mask1: static_cast<unsigned>(PrevWait->getOperand(i: 0).getImm()),
488 Mask2: static_cast<unsigned>(MaskOp.getImm())));
489 PrevWait->eraseFromParent();
490 Changed = true;
491 } else {
492 // Starting a new region using fresh write set.
493 WriteSet.reset();
494 }
495 CommitWrites(static_cast<unsigned>(MI.getOperand(i: 0).getImm()));
496 PrevWait = &MI;
497 continue;
498 }
499
500 // Do not optimize over branches or terminators
501 if (PrevWait && (MI.isCall() || MI.isReturn() || MI.isBranch() ||
502 MI.isTerminator())) {
503 PrevWait->moveBefore(MovePos: &MI);
504 PrevWait = nullptr;
505 Changed = true;
506 }
507 if (MI.isTerminator())
508 break;
509
510 // Note: test for meta instructions after terminators.
511 // Required to handle terminator meta instruction.
512 if (MI.isMetaInstruction())
513 continue;
514
515 const bool IsVALU = SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false);
516 const bool IsSALU = SIInstrInfo::isSALU(MI);
517 if (!IsVALU && !IsSALU)
518 continue;
519
520 for (const MachineOperand &Op : MI.operands()) {
521 if (!Op.isReg())
522 continue;
523 Register Reg = Op.getReg();
524 if (!TRI->isSGPRReg(MRI: *MRI, Reg))
525 continue;
526
527 std::optional<unsigned> RegNumber = sgprNumber(Reg, TRI: *TRI);
528 if (!RegNumber)
529 continue;
530 unsigned RegN = *RegNumber;
531 unsigned SGPRCount =
532 AMDGPU::getRegBitWidth(RC: *TRI->getRegClassForReg(MRI: *MRI, Reg)) / 32;
533
534 if (Op.isDef()) {
535 if (IsSALU)
536 PendingSALUWriteSet.set(I: RegN, E: RegN + SGPRCount);
537 else
538 PendingVALUWriteSet.set(I: RegN, E: RegN + SGPRCount);
539 continue;
540 }
541
542 if (PrevWait &&
543 WriteSet.find_prev(PriorTo: RegN + SGPRCount) >= (signed)RegN) {
544 // Move the wait to here, the last point it can be valid
545 PrevWait->moveBefore(MovePos: &MI);
546 PrevWait = nullptr;
547 Changed = true;
548 }
549 }
550 }
551 }
552 return Changed;
553 }
554
555 bool run(MachineFunction &MF) {
556 ST = &MF.getSubtarget<GCNSubtarget>();
557 if (!ST->hasVALUReadSGPRHazard() && !ST->hasVALUMaskWriteHazard())
558 return false;
559
560 // Parse settings
561 CullSGPRHazardsOnFunctionBoundary = GlobalCullSGPRHazardsOnFunctionBoundary;
562 CullSGPRHazardsAtMemWait = GlobalCullSGPRHazardsAtMemWait;
563 CullSGPRHazardsMemWaitThreshold = GlobalCullSGPRHazardsMemWaitThreshold;
564
565 if (!GlobalCullSGPRHazardsOnFunctionBoundary.getNumOccurrences())
566 CullSGPRHazardsOnFunctionBoundary =
567 MF.getFunction().hasFnAttribute(Kind: "amdgpu-sgpr-hazard-boundary-cull");
568 if (!GlobalCullSGPRHazardsAtMemWait.getNumOccurrences())
569 CullSGPRHazardsAtMemWait =
570 MF.getFunction().hasFnAttribute(Kind: "amdgpu-sgpr-hazard-mem-wait-cull");
571 if (!GlobalCullSGPRHazardsMemWaitThreshold.getNumOccurrences())
572 CullSGPRHazardsMemWaitThreshold =
573 static_cast<unsigned>(MF.getFunction().getFnAttributeAsParsedInteger(
574 Kind: "amdgpu-sgpr-hazard-mem-wait-cull-threshold",
575 Default: CullSGPRHazardsMemWaitThreshold));
576
577 TII = ST->getInstrInfo();
578 TRI = ST->getRegisterInfo();
579 MRI = &MF.getRegInfo();
580 DsNopCount = ST->isWave64() ? WAVE64_NOPS : WAVE32_NOPS;
581
582 // VALU mask write hazards have already been handled, but this pass
583 // performs a forward scan to optimize them.
584 if (ST->hasVALUMaskWriteHazard())
585 return ST->isWave64() ? runWaitMerging(MF) : false;
586
587 auto CallingConv = MF.getFunction().getCallingConv();
588 if (!AMDGPU::isEntryFunctionCC(CC: CallingConv) &&
589 !CullSGPRHazardsOnFunctionBoundary) {
590 // Callee must consider all SGPRs as tracked.
591 LLVM_DEBUG(dbgs() << "Is called function, track all SGPRs.\n");
592 MachineBasicBlock &EntryBlock = MF.front();
593 BlockState[&EntryBlock].In.Tracked.set();
594 }
595
596 // Calculate the hazard state for each basic block.
597 // Iterate until a fixed point is reached.
598 // Fixed point is guaranteed as merge function only ever increases
599 // the hazard set, and all backedges will cause a merge.
600 //
601 // Note: we have to take care of the entry block as this technically
602 // has an edge from outside the function. Failure to treat this as
603 // a merge could prevent fixed point being reached.
604 SetVector<MachineBasicBlock *> Worklist;
605 for (auto &MBB : reverse(C&: MF))
606 Worklist.insert(X: &MBB);
607 while (!Worklist.empty()) {
608 auto &MBB = *Worklist.pop_back_val();
609 bool Changed = runOnMachineBasicBlock(MBB, Emit: false);
610 if (Changed) {
611 // Note: take a copy of state here in case it is reallocated by map
612 HazardState NewState = BlockState[&MBB].Out;
613 // Propagate to all successor blocks
614 for (auto Succ : MBB.successors()) {
615 // We only need to merge hazards at CFG merge points.
616 auto &SuccState = BlockState[Succ];
617 if (Succ->getSinglePredecessor() && !Succ->isEntryBlock()) {
618 if (SuccState.In != NewState) {
619 SuccState.In = NewState;
620 Worklist.insert(X: Succ);
621 }
622 } else if (SuccState.In.merge(RHS: NewState)) {
623 Worklist.insert(X: Succ);
624 }
625 }
626 }
627 }
628
629 LLVM_DEBUG(dbgs() << "Emit s_wait_alu instructions\n");
630
631 // Final to emit wait instructions.
632 bool Changed = false;
633 for (auto &MBB : MF)
634 Changed |= runOnMachineBasicBlock(MBB, Emit: true);
635
636 BlockState.clear();
637 return Changed;
638 }
639};
640
641class AMDGPUWaitSGPRHazardsLegacy : public MachineFunctionPass {
642public:
643 static char ID;
644
645 AMDGPUWaitSGPRHazardsLegacy() : MachineFunctionPass(ID) {}
646
647 bool runOnMachineFunction(MachineFunction &MF) override {
648 return AMDGPUWaitSGPRHazards().run(MF);
649 }
650
651 void getAnalysisUsage(AnalysisUsage &AU) const override {
652 AU.setPreservesCFG();
653 MachineFunctionPass::getAnalysisUsage(AU);
654 }
655};
656
657} // namespace
658
659char AMDGPUWaitSGPRHazardsLegacy::ID = 0;
660
661char &llvm::AMDGPUWaitSGPRHazardsLegacyID = AMDGPUWaitSGPRHazardsLegacy::ID;
662
663INITIALIZE_PASS(AMDGPUWaitSGPRHazardsLegacy, DEBUG_TYPE,
664 "AMDGPU Insert waits for SGPR read hazards", false, false)
665
666PreservedAnalyses
667AMDGPUWaitSGPRHazardsPass::run(MachineFunction &MF,
668 MachineFunctionAnalysisManager &MFAM) {
669 if (AMDGPUWaitSGPRHazards().run(MF))
670 return getMachineFunctionPassPreservedAnalyses();
671 return PreservedAnalyses::all();
672}
673