1//===- AMDGPUWaitSGPRHazards.cpp - Insert waits for SGPR read hazards -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Insert s_wait_alu instructions to mitigate SGPR read hazards on GFX12.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUWaitSGPRHazards.h"
15#include "AMDGPU.h"
16#include "GCNSubtarget.h"
17#include "SIInstrInfo.h"
18#include "llvm/ADT/SmallBitVector.h"
19#include "llvm/TargetParser/AMDGPUTargetParser.h"
20
21using namespace llvm;
22
23#define DEBUG_TYPE "amdgpu-wait-sgpr-hazards"
24
25namespace {
26
27class AMDGPUWaitSGPRHazards {
28public:
29 const GCNSubtarget *ST;
30 const SIInstrInfo *TII;
31 const SIRegisterInfo *TRI;
32 const MachineRegisterInfo *MRI;
33 unsigned DsNopCount;
34
35 bool CullSGPRHazardsOnFunctionBoundary;
36 bool CullSGPRHazardsAtMemWait;
37 unsigned CullSGPRHazardsMemWaitThreshold;
38
39 AMDGPUWaitSGPRHazards() = default;
40
41 // Return the numeric ID 0-127 for a given SGPR.
42 static std::optional<unsigned> sgprNumber(Register Reg,
43 const SIRegisterInfo &TRI) {
44 switch (Reg) {
45 case AMDGPU::M0:
46 case AMDGPU::EXEC:
47 case AMDGPU::EXEC_LO:
48 case AMDGPU::EXEC_HI:
49 case AMDGPU::SGPR_NULL:
50 case AMDGPU::SGPR_NULL64:
51 case AMDGPU::SCC:
52 return {};
53 default:
54 break;
55 }
56 unsigned RegN = TRI.getHWRegIndex(Reg);
57 if (RegN > 127)
58 return {};
59 return RegN;
60 }
61
62 static inline bool isVCC(Register Reg) {
63 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI;
64 }
65
66 // Adjust global offsets for instructions bundled with S_GETPC_B64 after
67 // insertion of a new instruction.
68 static void updateGetPCBundle(MachineInstr *NewMI) {
69 if (!NewMI->isBundled())
70 return;
71
72 // Find start of bundle.
73 auto I = NewMI->getIterator();
74 while (I->isBundledWithPred())
75 I--;
76 if (I->isBundle())
77 I++;
78
79 // Bail if this is not an S_GETPC bundle.
80 if (I->getOpcode() != AMDGPU::S_GETPC_B64)
81 return;
82
83 // Update offsets of any references in the bundle.
84 const unsigned NewBytes = 4;
85 assert(NewMI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
86 "Unexpected instruction insertion in bundle");
87 auto NextMI = std::next(x: NewMI->getIterator());
88 auto End = NewMI->getParent()->end();
89 while (NextMI != End && NextMI->isBundledWithPred()) {
90 for (auto &Operand : NextMI->operands()) {
91 if (Operand.isGlobal())
92 Operand.setOffset(Operand.getOffset() + NewBytes);
93 }
94 NextMI++;
95 }
96 }
97
98 struct HazardState {
99 static constexpr unsigned None = 0;
100 static constexpr unsigned SALU = (1 << 0);
101 static constexpr unsigned VALU = (1 << 1);
102
103 std::bitset<64> Tracked; // SGPR banks ever read by VALU
104 std::bitset<128> SALUHazards; // SGPRs with uncommitted values from SALU
105 std::bitset<128> VALUHazards; // SGPRs with uncommitted values from VALU
106 unsigned VCCHazard = None; // Source of current VCC writes
107 bool ActiveFlat = false; // Has unwaited flat instructions
108
109 bool merge(const HazardState &RHS) {
110 HazardState Orig(*this);
111 *this |= RHS;
112 return (*this != Orig);
113 }
114
115 bool operator==(const HazardState &RHS) const {
116 return Tracked == RHS.Tracked && SALUHazards == RHS.SALUHazards &&
117 VALUHazards == RHS.VALUHazards && VCCHazard == RHS.VCCHazard &&
118 ActiveFlat == RHS.ActiveFlat;
119 }
120
121 bool operator!=(const HazardState &RHS) const { return !(*this == RHS); }
122
123 void operator|=(const HazardState &RHS) {
124 Tracked |= RHS.Tracked;
125 SALUHazards |= RHS.SALUHazards;
126 VALUHazards |= RHS.VALUHazards;
127 VCCHazard |= RHS.VCCHazard;
128 ActiveFlat |= RHS.ActiveFlat;
129 }
130 };
131
132 struct BlockHazardState {
133 HazardState In;
134 HazardState Out;
135 };
136
137 DenseMap<const MachineBasicBlock *, BlockHazardState> BlockState;
138
139 static constexpr unsigned WAVE32_NOPS = 4;
140 static constexpr unsigned WAVE64_NOPS = 8;
141
142 void insertHazardCull(MachineBasicBlock &MBB,
143 MachineBasicBlock::instr_iterator &MI) {
144 assert(!MI->isBundled());
145 unsigned Count = DsNopCount;
146 while (Count--)
147 BuildMI(BB&: MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::DS_NOP));
148 }
149
150 unsigned mergeMasks(unsigned Mask1, unsigned Mask2) {
151 unsigned Mask = AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: *ST);
152 Mask = AMDGPU::DepCtr::encodeFieldSaSdst(
153 Encoded: Mask, SaSdst: std::min(a: AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Mask1),
154 b: AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Mask2)));
155 Mask = AMDGPU::DepCtr::encodeFieldVaVcc(
156 Encoded: Mask, VaVcc: std::min(a: AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Mask1),
157 b: AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Mask2)));
158 Mask = AMDGPU::DepCtr::encodeFieldVmVsrc(
159 Encoded: Mask, VmVsrc: std::min(a: AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: Mask1),
160 b: AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: Mask2)));
161 Mask = AMDGPU::DepCtr::encodeFieldVaSdst(
162 Encoded: Mask, VaSdst: std::min(a: AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Mask1),
163 b: AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Mask2)));
164 Mask = AMDGPU::DepCtr::encodeFieldVaVdst(
165 Encoded: Mask, VaVdst: std::min(a: AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: Mask1),
166 b: AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: Mask2)));
167 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: ST->getCPU());
168 Mask = AMDGPU::DepCtr::encodeFieldHoldCnt(
169 Encoded: Mask,
170 HoldCnt: std::min(a: AMDGPU::DepCtr::decodeFieldHoldCnt(Encoded: Mask1, Version),
171 b: AMDGPU::DepCtr::decodeFieldHoldCnt(Encoded: Mask2, Version)),
172 Version);
173 Mask = AMDGPU::DepCtr::encodeFieldVaSsrc(
174 Encoded: Mask, VaSsrc: std::min(a: AMDGPU::DepCtr::decodeFieldVaSsrc(Encoded: Mask1),
175 b: AMDGPU::DepCtr::decodeFieldVaSsrc(Encoded: Mask2)));
176 return Mask;
177 }
178
179 bool mergeConsecutiveWaitAlus(MachineBasicBlock::instr_iterator &MI,
180 unsigned Mask) {
181 auto MBB = MI->getParent();
182 if (MI == MBB->instr_begin())
183 return false;
184
185 auto It = prev_nodbg(It: MI, Begin: MBB->instr_begin());
186 if (It->getOpcode() != AMDGPU::S_WAITCNT_DEPCTR)
187 return false;
188
189 It->getOperand(i: 0).setImm(
190 mergeMasks(Mask1: Mask, Mask2: static_cast<unsigned>(It->getOperand(i: 0).getImm())));
191 return true;
192 }
193
194 bool runOnMachineBasicBlock(MachineBasicBlock &MBB, bool Emit) {
195 enum { WA_VALU = 0x1, WA_SALU = 0x2, WA_VCC = 0x4 };
196
197 HazardState State = BlockState[&MBB].In;
198 SmallSet<Register, 8> SeenRegs;
199 bool Emitted = false;
200 unsigned DsNops = 0;
201
202 for (MachineBasicBlock::instr_iterator MI = MBB.instr_begin(),
203 E = MBB.instr_end();
204 MI != E; ++MI) {
205 if (MI->isMetaInstruction())
206 continue;
207
208 // Clear tracked SGPRs if sufficient DS_NOPs occur
209 if (MI->getOpcode() == AMDGPU::DS_NOP) {
210 if (++DsNops >= DsNopCount)
211 State.Tracked.reset();
212 continue;
213 }
214 DsNops = 0;
215
216 // Snoop FLAT instructions to avoid adding culls before scratch/lds loads.
217 // Culls could be disproportionate in cost to load time.
218 if (SIInstrInfo::isFLAT(MI: *MI) && !SIInstrInfo::isFLATGlobal(MI: *MI))
219 State.ActiveFlat = true;
220
221 // SMEM or VMEM clears hazards
222 // FIXME: adapt to add FLAT without VALU (so !isLDSDMA())?
223 if ((SIInstrInfo::isVMEM(MI: *MI) && !SIInstrInfo::isFLAT(MI: *MI)) ||
224 SIInstrInfo::isSMRD(MI: *MI)) {
225 State.VCCHazard = HazardState::None;
226 State.SALUHazards.reset();
227 State.VALUHazards.reset();
228 continue;
229 }
230
231 // Existing S_WAITALU can clear hazards
232 if (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR) {
233 unsigned int Mask = static_cast<unsigned>(MI->getOperand(i: 0).getImm());
234 if (AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Mask) == 0)
235 State.VCCHazard &= ~HazardState::VALU;
236 if (AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Mask) == 0) {
237 State.SALUHazards.reset();
238 State.VCCHazard &= ~HazardState::SALU;
239 }
240 if (AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Mask) == 0)
241 State.VALUHazards.reset();
242 continue;
243 }
244
245 // Snoop counter waits to insert culls
246 if (CullSGPRHazardsAtMemWait &&
247 (MI->getOpcode() == AMDGPU::S_WAIT_LOADCNT ||
248 MI->getOpcode() == AMDGPU::S_WAIT_SAMPLECNT ||
249 MI->getOpcode() == AMDGPU::S_WAIT_BVHCNT) &&
250 (MI->getOperand(i: 0).isImm() && MI->getOperand(i: 0).getImm() == 0) &&
251 (State.Tracked.count() >= CullSGPRHazardsMemWaitThreshold)) {
252 if (MI->getOpcode() == AMDGPU::S_WAIT_LOADCNT && State.ActiveFlat) {
253 State.ActiveFlat = false;
254 } else {
255 State.Tracked.reset();
256 if (Emit)
257 insertHazardCull(MBB, MI);
258 continue;
259 }
260 }
261
262 // Process only VALUs and SALUs
263 bool IsVALU = SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/false);
264 bool IsSALU = SIInstrInfo::isSALU(MI: *MI);
265 if (!IsVALU && !IsSALU)
266 continue;
267
268 unsigned Wait = 0;
269
270 auto processOperand = [&](const MachineOperand &Op, bool IsUse) {
271 if (!Op.isReg())
272 return;
273 Register Reg = Op.getReg();
274 assert(!Op.getSubReg());
275 if (!TRI->isSGPRReg(MRI: *MRI, Reg))
276 return;
277
278 // Only visit each register once
279 if (!SeenRegs.insert(V: Reg).second)
280 return;
281
282 auto RegNumber = sgprNumber(Reg, TRI: *TRI);
283 if (!RegNumber)
284 return;
285
286 // Track SGPRs by pair -- numeric ID of an 64b SGPR pair.
287 // i.e. SGPR0 = SGPR0_SGPR1 = 0, SGPR3 = SGPR2_SGPR3 = 1, etc
288 unsigned RegN = *RegNumber;
289 unsigned PairN = (RegN >> 1) & 0x3f;
290
291 // Read/write of untracked register is safe; but must record any new
292 // reads.
293 if (!State.Tracked[PairN]) {
294 if (IsVALU && IsUse)
295 State.Tracked.set(position: PairN);
296 return;
297 }
298
299 unsigned SGPRCount =
300 AMDGPU::getRegBitWidth(RC: *TRI->getRegClassForReg(MRI: *MRI, Reg)) / 32;
301
302 if (IsUse) {
303 // SALU reading SGPR clears VALU hazards
304 if (IsSALU) {
305 if (isVCC(Reg)) {
306 if (State.VCCHazard & HazardState::VALU)
307 State.VCCHazard = HazardState::None;
308 } else {
309 State.VALUHazards.reset();
310 }
311 }
312 // Compute required waits
313 for (unsigned RegIdx = 0; RegIdx < SGPRCount; ++RegIdx) {
314 Wait |= State.SALUHazards[RegN + RegIdx] ? WA_SALU : 0;
315 Wait |= IsVALU && State.VALUHazards[RegN + RegIdx] ? WA_VALU : 0;
316 }
317 if (isVCC(Reg) && State.VCCHazard) {
318 // Note: it's possible for both SALU and VALU to exist if VCC
319 // was updated differently by merged predecessors.
320 if (State.VCCHazard & HazardState::SALU)
321 Wait |= WA_SALU;
322 if (State.VCCHazard & HazardState::VALU)
323 Wait |= WA_VCC;
324 }
325 } else {
326 // Update hazards
327 if (isVCC(Reg)) {
328 State.VCCHazard = IsSALU ? HazardState::SALU : HazardState::VALU;
329 } else {
330 for (unsigned RegIdx = 0; RegIdx < SGPRCount; ++RegIdx) {
331 if (IsSALU)
332 State.SALUHazards.set(position: RegN + RegIdx);
333 else
334 State.VALUHazards.set(position: RegN + RegIdx);
335 }
336 }
337 }
338 };
339
340 const bool IsSetPC =
341 (MI->isCall() || MI->isReturn() || MI->isIndirectBranch()) &&
342 MI->getOpcode() != AMDGPU::S_ENDPGM &&
343 MI->getOpcode() != AMDGPU::S_ENDPGM_SAVED;
344
345 // Only consider implicit VCC specified by instruction descriptor.
346 const bool HasImplicitVCC =
347 llvm::any_of(Range: MI->getDesc().implicit_uses(), P: isVCC) ||
348 llvm::any_of(Range: MI->getDesc().implicit_defs(), P: isVCC);
349
350 if (IsSetPC) {
351 // All SGPR writes before a call/return must be flushed as the
352 // callee/caller will not will not see the hazard chain.
353 if (State.VCCHazard & HazardState::VALU)
354 Wait |= WA_VCC;
355 if (State.SALUHazards.any() || (State.VCCHazard & HazardState::SALU))
356 Wait |= WA_SALU;
357 if (State.VALUHazards.any())
358 Wait |= WA_VALU;
359 if (CullSGPRHazardsOnFunctionBoundary && State.Tracked.any()) {
360 State.Tracked.reset();
361 if (Emit)
362 insertHazardCull(MBB, MI);
363 }
364 } else {
365 // Process uses to determine required wait.
366 SeenRegs.clear();
367 for (const MachineOperand &Op : MI->all_uses()) {
368 if (Op.isImplicit() &&
369 (!HasImplicitVCC || !Op.isReg() || !isVCC(Reg: Op.getReg())))
370 continue;
371 processOperand(Op, true);
372 }
373 }
374
375 // Apply wait
376 if (Wait) {
377 unsigned Mask = AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: *ST);
378 if (Wait & WA_VCC) {
379 State.VCCHazard &= ~HazardState::VALU;
380 Mask = AMDGPU::DepCtr::encodeFieldVaVcc(Encoded: Mask, VaVcc: 0);
381 }
382 if (Wait & WA_SALU) {
383 State.SALUHazards.reset();
384 State.VCCHazard &= ~HazardState::SALU;
385 Mask = AMDGPU::DepCtr::encodeFieldSaSdst(Encoded: Mask, SaSdst: 0);
386 }
387 if (Wait & WA_VALU) {
388 State.VALUHazards.reset();
389 Mask = AMDGPU::DepCtr::encodeFieldVaSdst(Encoded: Mask, VaSdst: 0);
390 }
391 if (Emit) {
392 if (!mergeConsecutiveWaitAlus(MI, Mask)) {
393 auto NewMI = BuildMI(BB&: MBB, I: MI, MIMD: MI->getDebugLoc(),
394 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
395 .addImm(Val: Mask);
396 updateGetPCBundle(NewMI);
397 }
398 Emitted = true;
399 }
400 }
401
402 // On return from a call SGPR state is unknown, so all potential hazards.
403 if (MI->isCall() && !CullSGPRHazardsOnFunctionBoundary)
404 State.Tracked.set();
405
406 // Update hazards based on defs.
407 SeenRegs.clear();
408 for (const MachineOperand &Op : MI->all_defs()) {
409 if (Op.isImplicit() &&
410 (!HasImplicitVCC || !Op.isReg() || !isVCC(Reg: Op.getReg())))
411 continue;
412 processOperand(Op, false);
413 }
414 }
415
416 BlockHazardState &BS = BlockState[&MBB];
417 bool Changed = State != BS.Out;
418 if (Emit) {
419 assert(!Changed && "Hazard state should not change on emit pass");
420 return Emitted;
421 }
422 if (Changed)
423 BS.Out = State;
424 return Changed;
425 }
426
427 bool runWaitMerging(MachineFunction &MF) {
428 // Perform per-block merging of existing s_waitcnt_depctr instructions.
429 // Track set of SGPR writes before a given wait instruction, and search
430 // for reads of these SGPRs.
431 // Move the wait to just before the read to improve pipelining.
432 // If no related reads occur before subsequent wait then merged waits.
433 const unsigned ConstantMaskBits = AMDGPU::DepCtr::encodeFieldSaSdst(
434 Encoded: AMDGPU::DepCtr::encodeFieldVaSdst(
435 Encoded: AMDGPU::DepCtr::encodeFieldVaVcc(VaVcc: 0, STI: *ST), VaSdst: 0),
436 SaSdst: 0);
437 const unsigned VccLoIdx = *sgprNumber(Reg: AMDGPU::VCC_LO, TRI: *TRI);
438 const unsigned VccHiIdx = *sgprNumber(Reg: AMDGPU::VCC_HI, TRI: *TRI);
439 bool Changed = false;
440 for (MachineBasicBlock &MBB : MF) {
441 SmallBitVector WriteSet(128), PendingSALUWriteSet(128),
442 PendingVALUWriteSet(128);
443 MachineInstr *PrevWait = nullptr;
444
445 auto CommitWrites = [&](unsigned Mask) {
446 if (!AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Mask))
447 WriteSet |= PendingSALUWriteSet;
448 bool VccLoBit = WriteSet[VccLoIdx];
449 bool VccHiBit = WriteSet[VccHiIdx];
450 if (!AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Mask)) {
451 // Apply pending VALU set minus VCC bits
452 WriteSet |= PendingVALUWriteSet;
453 WriteSet[VccLoIdx] = VccLoBit;
454 WriteSet[VccHiIdx] = VccHiBit;
455 }
456 if (!AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Mask)) {
457 WriteSet[VccLoIdx] = VccLoBit || PendingVALUWriteSet[VccLoIdx];
458 WriteSet[VccHiIdx] = VccHiBit || PendingVALUWriteSet[VccHiIdx];
459 }
460 // Clear all pending writes
461 PendingSALUWriteSet.reset();
462 PendingVALUWriteSet.reset();
463 };
464
465 for (MachineInstr &MI : MBB) {
466 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
467 (MI.getOperand(i: 0).getImm() & ConstantMaskBits) ==
468 ConstantMaskBits) {
469 if (PrevWait) {
470 // Merge previous wait into this one.
471 MachineOperand &MaskOp = MI.getOperand(i: 0);
472 MaskOp.setImm(mergeMasks(
473 Mask1: static_cast<unsigned>(PrevWait->getOperand(i: 0).getImm()),
474 Mask2: static_cast<unsigned>(MaskOp.getImm())));
475 PrevWait->eraseFromParent();
476 Changed = true;
477 } else {
478 // Starting a new region using fresh write set.
479 WriteSet.reset();
480 }
481 CommitWrites(static_cast<unsigned>(MI.getOperand(i: 0).getImm()));
482 PrevWait = &MI;
483 continue;
484 }
485
486 // Do not optimize over branches or terminators
487 if (PrevWait && (MI.isCall() || MI.isReturn() || MI.isBranch() ||
488 MI.isTerminator())) {
489 PrevWait->moveBefore(MovePos: &MI);
490 PrevWait = nullptr;
491 Changed = true;
492 }
493 if (MI.isTerminator())
494 break;
495
496 // Note: test for meta instructions after terminators.
497 // Required to handle terminator meta instruction.
498 if (MI.isMetaInstruction())
499 continue;
500
501 const bool IsVALU = SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false);
502 const bool IsSALU = SIInstrInfo::isSALU(MI);
503 if (!IsVALU && !IsSALU)
504 continue;
505
506 for (const MachineOperand &Op : MI.operands()) {
507 if (!Op.isReg())
508 continue;
509 Register Reg = Op.getReg();
510 if (!TRI->isSGPRReg(MRI: *MRI, Reg))
511 continue;
512
513 std::optional<unsigned> RegNumber = sgprNumber(Reg, TRI: *TRI);
514 if (!RegNumber)
515 continue;
516 unsigned RegN = *RegNumber;
517 unsigned SGPRCount =
518 AMDGPU::getRegBitWidth(RC: *TRI->getRegClassForReg(MRI: *MRI, Reg)) / 32;
519
520 if (Op.isDef()) {
521 if (IsSALU)
522 PendingSALUWriteSet.set(I: RegN, E: RegN + SGPRCount);
523 else
524 PendingVALUWriteSet.set(I: RegN, E: RegN + SGPRCount);
525 continue;
526 }
527
528 if (PrevWait &&
529 WriteSet.find_prev(PriorTo: RegN + SGPRCount) >= (signed)RegN) {
530 // Move the wait to here, the last point it can be valid
531 PrevWait->moveBefore(MovePos: &MI);
532 PrevWait = nullptr;
533 Changed = true;
534 }
535 }
536 }
537 }
538 return Changed;
539 }
540
541 bool run(MachineFunction &MF) {
542 ST = &MF.getSubtarget<GCNSubtarget>();
543 if (!ST->hasVALUReadSGPRHazard() && !ST->hasVALUMaskWriteHazard())
544 return false;
545
546 // Parse settings
547 const AMDGPUOptions &CLOpts = ST->getCLOpts();
548 CullSGPRHazardsOnFunctionBoundary = valueOr(
549 X: CLOpts.sgpr_hazard_boundary_cull,
550 Default: MF.getFunction().hasFnAttribute(Kind: "amdgpu-sgpr-hazard-boundary-cull"));
551 CullSGPRHazardsAtMemWait = valueOr(
552 X: CLOpts.sgpr_hazard_mem_wait_cull,
553 Default: MF.getFunction().hasFnAttribute(Kind: "amdgpu-sgpr-hazard-mem-wait-cull"));
554 CullSGPRHazardsMemWaitThreshold =
555 CLOpts.sgpr_hazard_mem_wait_cull_threshold.value_or(
556 u: MF.getFunction().getFnAttributeAsParsedInteger(
557 Kind: "amdgpu-sgpr-hazard-mem-wait-cull-threshold", Default: 8));
558
559 TII = ST->getInstrInfo();
560 TRI = ST->getRegisterInfo();
561 MRI = &MF.getRegInfo();
562 DsNopCount = ST->isWave64() ? WAVE64_NOPS : WAVE32_NOPS;
563
564 // VALU mask write hazards have already been handled, but this pass
565 // performs a forward scan to optimize them.
566 if (ST->hasVALUMaskWriteHazard())
567 return ST->isWave64() ? runWaitMerging(MF) : false;
568
569 auto CallingConv = MF.getFunction().getCallingConv();
570 if (!AMDGPU::isEntryFunctionCC(CC: CallingConv) &&
571 !CullSGPRHazardsOnFunctionBoundary) {
572 // Callee must consider all SGPRs as tracked.
573 LLVM_DEBUG(dbgs() << "Is called function, track all SGPRs.\n");
574 MachineBasicBlock &EntryBlock = MF.front();
575 BlockState[&EntryBlock].In.Tracked.set();
576 }
577
578 // Calculate the hazard state for each basic block.
579 // Iterate until a fixed point is reached.
580 // Fixed point is guaranteed as merge function only ever increases
581 // the hazard set, and all backedges will cause a merge.
582 //
583 // Note: we have to take care of the entry block as this technically
584 // has an edge from outside the function. Failure to treat this as
585 // a merge could prevent fixed point being reached.
586 SetVector<MachineBasicBlock *> Worklist;
587 for (auto &MBB : reverse(C&: MF))
588 Worklist.insert(X: &MBB);
589 while (!Worklist.empty()) {
590 auto &MBB = *Worklist.pop_back_val();
591 bool Changed = runOnMachineBasicBlock(MBB, Emit: false);
592 if (Changed) {
593 // Note: take a copy of state here in case it is reallocated by map
594 HazardState NewState = BlockState[&MBB].Out;
595 // Propagate to all successor blocks
596 for (auto Succ : MBB.successors()) {
597 // We only need to merge hazards at CFG merge points.
598 auto &SuccState = BlockState[Succ];
599 if (Succ->getSinglePredecessor() && !Succ->isEntryBlock()) {
600 if (SuccState.In != NewState) {
601 SuccState.In = NewState;
602 Worklist.insert(X: Succ);
603 }
604 } else if (SuccState.In.merge(RHS: NewState)) {
605 Worklist.insert(X: Succ);
606 }
607 }
608 }
609 }
610
611 LLVM_DEBUG(dbgs() << "Emit s_wait_alu instructions\n");
612
613 // Final to emit wait instructions.
614 bool Changed = false;
615 for (auto &MBB : MF)
616 Changed |= runOnMachineBasicBlock(MBB, Emit: true);
617
618 BlockState.clear();
619 return Changed;
620 }
621};
622
623class AMDGPUWaitSGPRHazardsLegacy : public MachineFunctionPass {
624public:
625 static char ID;
626
627 AMDGPUWaitSGPRHazardsLegacy() : MachineFunctionPass(ID) {}
628
629 bool runOnMachineFunction(MachineFunction &MF) override {
630 return AMDGPUWaitSGPRHazards().run(MF);
631 }
632
633 void getAnalysisUsage(AnalysisUsage &AU) const override {
634 AU.setPreservesCFG();
635 MachineFunctionPass::getAnalysisUsage(AU);
636 }
637};
638
639} // namespace
640
641char AMDGPUWaitSGPRHazardsLegacy::ID = 0;
642
643char &llvm::AMDGPUWaitSGPRHazardsLegacyID = AMDGPUWaitSGPRHazardsLegacy::ID;
644
645INITIALIZE_PASS(AMDGPUWaitSGPRHazardsLegacy, DEBUG_TYPE,
646 "AMDGPU Insert waits for SGPR read hazards", false, false)
647
648PreservedAnalyses
649AMDGPUWaitSGPRHazardsPass::run(MachineFunction &MF,
650 MachineFunctionAnalysisManager &MFAM) {
651 if (AMDGPUWaitSGPRHazards().run(MF))
652 return getMachineFunctionPassPreservedAnalyses();
653 return PreservedAnalyses::all();
654}
655