1//===- AMDGPUWaitSGPRHazards.cpp - Insert waits for SGPR read hazards -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Insert s_wait_alu instructions to mitigate SGPR read hazards on GFX12.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUWaitSGPRHazards.h"
15#include "AMDGPU.h"
16#include "GCNSubtarget.h"
17#include "SIInstrInfo.h"
18#include "llvm/ADT/SmallBitVector.h"
19#include "llvm/TargetParser/AMDGPUTargetParser.h"
20
21using namespace llvm;
22
23#define DEBUG_TYPE "amdgpu-wait-sgpr-hazards"
24
25static cl::opt<bool> GlobalCullSGPRHazardsOnFunctionBoundary(
26 "amdgpu-sgpr-hazard-boundary-cull", cl::init(Val: false), cl::Hidden,
27 cl::desc("Cull hazards on function boundaries"));
28
29static cl::opt<bool>
30 GlobalCullSGPRHazardsAtMemWait("amdgpu-sgpr-hazard-mem-wait-cull",
31 cl::init(Val: false), cl::Hidden,
32 cl::desc("Cull hazards on memory waits"));
33
34static cl::opt<unsigned> GlobalCullSGPRHazardsMemWaitThreshold(
35 "amdgpu-sgpr-hazard-mem-wait-cull-threshold", cl::init(Val: 8), cl::Hidden,
36 cl::desc("Number of tracked SGPRs before initiating hazard cull on memory "
37 "wait"));
38
39namespace {
40
41class AMDGPUWaitSGPRHazards {
42public:
43 const GCNSubtarget *ST;
44 const SIInstrInfo *TII;
45 const SIRegisterInfo *TRI;
46 const MachineRegisterInfo *MRI;
47 unsigned DsNopCount;
48
49 bool CullSGPRHazardsOnFunctionBoundary;
50 bool CullSGPRHazardsAtMemWait;
51 unsigned CullSGPRHazardsMemWaitThreshold;
52
53 AMDGPUWaitSGPRHazards() = default;
54
55 // Return the numeric ID 0-127 for a given SGPR.
56 static std::optional<unsigned> sgprNumber(Register Reg,
57 const SIRegisterInfo &TRI) {
58 switch (Reg) {
59 case AMDGPU::M0:
60 case AMDGPU::EXEC:
61 case AMDGPU::EXEC_LO:
62 case AMDGPU::EXEC_HI:
63 case AMDGPU::SGPR_NULL:
64 case AMDGPU::SGPR_NULL64:
65 case AMDGPU::SCC:
66 return {};
67 default:
68 break;
69 }
70 unsigned RegN = TRI.getHWRegIndex(Reg);
71 if (RegN > 127)
72 return {};
73 return RegN;
74 }
75
76 static inline bool isVCC(Register Reg) {
77 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI;
78 }
79
80 // Adjust global offsets for instructions bundled with S_GETPC_B64 after
81 // insertion of a new instruction.
82 static void updateGetPCBundle(MachineInstr *NewMI) {
83 if (!NewMI->isBundled())
84 return;
85
86 // Find start of bundle.
87 auto I = NewMI->getIterator();
88 while (I->isBundledWithPred())
89 I--;
90 if (I->isBundle())
91 I++;
92
93 // Bail if this is not an S_GETPC bundle.
94 if (I->getOpcode() != AMDGPU::S_GETPC_B64)
95 return;
96
97 // Update offsets of any references in the bundle.
98 const unsigned NewBytes = 4;
99 assert(NewMI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
100 "Unexpected instruction insertion in bundle");
101 auto NextMI = std::next(x: NewMI->getIterator());
102 auto End = NewMI->getParent()->end();
103 while (NextMI != End && NextMI->isBundledWithPred()) {
104 for (auto &Operand : NextMI->operands()) {
105 if (Operand.isGlobal())
106 Operand.setOffset(Operand.getOffset() + NewBytes);
107 }
108 NextMI++;
109 }
110 }
111
112 struct HazardState {
113 static constexpr unsigned None = 0;
114 static constexpr unsigned SALU = (1 << 0);
115 static constexpr unsigned VALU = (1 << 1);
116
117 std::bitset<64> Tracked; // SGPR banks ever read by VALU
118 std::bitset<128> SALUHazards; // SGPRs with uncommitted values from SALU
119 std::bitset<128> VALUHazards; // SGPRs with uncommitted values from VALU
120 unsigned VCCHazard = None; // Source of current VCC writes
121 bool ActiveFlat = false; // Has unwaited flat instructions
122
123 bool merge(const HazardState &RHS) {
124 HazardState Orig(*this);
125 *this |= RHS;
126 return (*this != Orig);
127 }
128
129 bool operator==(const HazardState &RHS) const {
130 return Tracked == RHS.Tracked && SALUHazards == RHS.SALUHazards &&
131 VALUHazards == RHS.VALUHazards && VCCHazard == RHS.VCCHazard &&
132 ActiveFlat == RHS.ActiveFlat;
133 }
134
135 bool operator!=(const HazardState &RHS) const { return !(*this == RHS); }
136
137 void operator|=(const HazardState &RHS) {
138 Tracked |= RHS.Tracked;
139 SALUHazards |= RHS.SALUHazards;
140 VALUHazards |= RHS.VALUHazards;
141 VCCHazard |= RHS.VCCHazard;
142 ActiveFlat |= RHS.ActiveFlat;
143 }
144 };
145
146 struct BlockHazardState {
147 HazardState In;
148 HazardState Out;
149 };
150
151 DenseMap<const MachineBasicBlock *, BlockHazardState> BlockState;
152
153 static constexpr unsigned WAVE32_NOPS = 4;
154 static constexpr unsigned WAVE64_NOPS = 8;
155
156 void insertHazardCull(MachineBasicBlock &MBB,
157 MachineBasicBlock::instr_iterator &MI) {
158 assert(!MI->isBundled());
159 unsigned Count = DsNopCount;
160 while (Count--)
161 BuildMI(BB&: MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::DS_NOP));
162 }
163
164 unsigned mergeMasks(unsigned Mask1, unsigned Mask2) {
165 unsigned Mask = AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: *ST);
166 Mask = AMDGPU::DepCtr::encodeFieldSaSdst(
167 Encoded: Mask, SaSdst: std::min(a: AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Mask1),
168 b: AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Mask2)));
169 Mask = AMDGPU::DepCtr::encodeFieldVaVcc(
170 Encoded: Mask, VaVcc: std::min(a: AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Mask1),
171 b: AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Mask2)));
172 Mask = AMDGPU::DepCtr::encodeFieldVmVsrc(
173 Encoded: Mask, VmVsrc: std::min(a: AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: Mask1),
174 b: AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: Mask2)));
175 Mask = AMDGPU::DepCtr::encodeFieldVaSdst(
176 Encoded: Mask, VaSdst: std::min(a: AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Mask1),
177 b: AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Mask2)));
178 Mask = AMDGPU::DepCtr::encodeFieldVaVdst(
179 Encoded: Mask, VaVdst: std::min(a: AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: Mask1),
180 b: AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: Mask2)));
181 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: ST->getCPU());
182 Mask = AMDGPU::DepCtr::encodeFieldHoldCnt(
183 Encoded: Mask,
184 HoldCnt: std::min(a: AMDGPU::DepCtr::decodeFieldHoldCnt(Encoded: Mask1, Version),
185 b: AMDGPU::DepCtr::decodeFieldHoldCnt(Encoded: Mask2, Version)),
186 Version);
187 Mask = AMDGPU::DepCtr::encodeFieldVaSsrc(
188 Encoded: Mask, VaSsrc: std::min(a: AMDGPU::DepCtr::decodeFieldVaSsrc(Encoded: Mask1),
189 b: AMDGPU::DepCtr::decodeFieldVaSsrc(Encoded: Mask2)));
190 return Mask;
191 }
192
193 bool mergeConsecutiveWaitAlus(MachineBasicBlock::instr_iterator &MI,
194 unsigned Mask) {
195 auto MBB = MI->getParent();
196 if (MI == MBB->instr_begin())
197 return false;
198
199 auto It = prev_nodbg(It: MI, Begin: MBB->instr_begin());
200 if (It->getOpcode() != AMDGPU::S_WAITCNT_DEPCTR)
201 return false;
202
203 It->getOperand(i: 0).setImm(mergeMasks(Mask1: Mask, Mask2: It->getOperand(i: 0).getImm()));
204 return true;
205 }
206
207 bool runOnMachineBasicBlock(MachineBasicBlock &MBB, bool Emit) {
208 enum { WA_VALU = 0x1, WA_SALU = 0x2, WA_VCC = 0x4 };
209
210 HazardState State = BlockState[&MBB].In;
211 SmallSet<Register, 8> SeenRegs;
212 bool Emitted = false;
213 unsigned DsNops = 0;
214
215 for (MachineBasicBlock::instr_iterator MI = MBB.instr_begin(),
216 E = MBB.instr_end();
217 MI != E; ++MI) {
218 if (MI->isMetaInstruction())
219 continue;
220
221 // Clear tracked SGPRs if sufficient DS_NOPs occur
222 if (MI->getOpcode() == AMDGPU::DS_NOP) {
223 if (++DsNops >= DsNopCount)
224 State.Tracked.reset();
225 continue;
226 }
227 DsNops = 0;
228
229 // Snoop FLAT instructions to avoid adding culls before scratch/lds loads.
230 // Culls could be disproportionate in cost to load time.
231 if (SIInstrInfo::isFLAT(MI: *MI) && !SIInstrInfo::isFLATGlobal(MI: *MI))
232 State.ActiveFlat = true;
233
234 // SMEM or VMEM clears hazards
235 // FIXME: adapt to add FLAT without VALU (so !isLDSDMA())?
236 if ((SIInstrInfo::isVMEM(MI: *MI) && !SIInstrInfo::isFLAT(MI: *MI)) ||
237 SIInstrInfo::isSMRD(MI: *MI)) {
238 State.VCCHazard = HazardState::None;
239 State.SALUHazards.reset();
240 State.VALUHazards.reset();
241 continue;
242 }
243
244 // Existing S_WAITALU can clear hazards
245 if (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR) {
246 unsigned int Mask = MI->getOperand(i: 0).getImm();
247 if (AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Mask) == 0)
248 State.VCCHazard &= ~HazardState::VALU;
249 if (AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Mask) == 0) {
250 State.SALUHazards.reset();
251 State.VCCHazard &= ~HazardState::SALU;
252 }
253 if (AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Mask) == 0)
254 State.VALUHazards.reset();
255 continue;
256 }
257
258 // Snoop counter waits to insert culls
259 if (CullSGPRHazardsAtMemWait &&
260 (MI->getOpcode() == AMDGPU::S_WAIT_LOADCNT ||
261 MI->getOpcode() == AMDGPU::S_WAIT_SAMPLECNT ||
262 MI->getOpcode() == AMDGPU::S_WAIT_BVHCNT) &&
263 (MI->getOperand(i: 0).isImm() && MI->getOperand(i: 0).getImm() == 0) &&
264 (State.Tracked.count() >= CullSGPRHazardsMemWaitThreshold)) {
265 if (MI->getOpcode() == AMDGPU::S_WAIT_LOADCNT && State.ActiveFlat) {
266 State.ActiveFlat = false;
267 } else {
268 State.Tracked.reset();
269 if (Emit)
270 insertHazardCull(MBB, MI);
271 continue;
272 }
273 }
274
275 // Process only VALUs and SALUs
276 bool IsVALU = SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/false);
277 bool IsSALU = SIInstrInfo::isSALU(MI: *MI);
278 if (!IsVALU && !IsSALU)
279 continue;
280
281 unsigned Wait = 0;
282
283 auto processOperand = [&](const MachineOperand &Op, bool IsUse) {
284 if (!Op.isReg())
285 return;
286 Register Reg = Op.getReg();
287 assert(!Op.getSubReg());
288 if (!TRI->isSGPRReg(MRI: *MRI, Reg))
289 return;
290
291 // Only visit each register once
292 if (!SeenRegs.insert(V: Reg).second)
293 return;
294
295 auto RegNumber = sgprNumber(Reg, TRI: *TRI);
296 if (!RegNumber)
297 return;
298
299 // Track SGPRs by pair -- numeric ID of an 64b SGPR pair.
300 // i.e. SGPR0 = SGPR0_SGPR1 = 0, SGPR3 = SGPR2_SGPR3 = 1, etc
301 unsigned RegN = *RegNumber;
302 unsigned PairN = (RegN >> 1) & 0x3f;
303
304 // Read/write of untracked register is safe; but must record any new
305 // reads.
306 if (!State.Tracked[PairN]) {
307 if (IsVALU && IsUse)
308 State.Tracked.set(position: PairN);
309 return;
310 }
311
312 uint8_t SGPRCount =
313 AMDGPU::getRegBitWidth(RC: *TRI->getRegClassForReg(MRI: *MRI, Reg)) / 32;
314
315 if (IsUse) {
316 // SALU reading SGPR clears VALU hazards
317 if (IsSALU) {
318 if (isVCC(Reg)) {
319 if (State.VCCHazard & HazardState::VALU)
320 State.VCCHazard = HazardState::None;
321 } else {
322 State.VALUHazards.reset();
323 }
324 }
325 // Compute required waits
326 for (uint8_t RegIdx = 0; RegIdx < SGPRCount; ++RegIdx) {
327 Wait |= State.SALUHazards[RegN + RegIdx] ? WA_SALU : 0;
328 Wait |= IsVALU && State.VALUHazards[RegN + RegIdx] ? WA_VALU : 0;
329 }
330 if (isVCC(Reg) && State.VCCHazard) {
331 // Note: it's possible for both SALU and VALU to exist if VCC
332 // was updated differently by merged predecessors.
333 if (State.VCCHazard & HazardState::SALU)
334 Wait |= WA_SALU;
335 if (State.VCCHazard & HazardState::VALU)
336 Wait |= WA_VCC;
337 }
338 } else {
339 // Update hazards
340 if (isVCC(Reg)) {
341 State.VCCHazard = IsSALU ? HazardState::SALU : HazardState::VALU;
342 } else {
343 for (uint8_t RegIdx = 0; RegIdx < SGPRCount; ++RegIdx) {
344 if (IsSALU)
345 State.SALUHazards.set(position: RegN + RegIdx);
346 else
347 State.VALUHazards.set(position: RegN + RegIdx);
348 }
349 }
350 }
351 };
352
353 const bool IsSetPC =
354 (MI->isCall() || MI->isReturn() || MI->isIndirectBranch()) &&
355 MI->getOpcode() != AMDGPU::S_ENDPGM &&
356 MI->getOpcode() != AMDGPU::S_ENDPGM_SAVED;
357
358 // Only consider implicit VCC specified by instruction descriptor.
359 const bool HasImplicitVCC =
360 llvm::any_of(Range: MI->getDesc().implicit_uses(), P: isVCC) ||
361 llvm::any_of(Range: MI->getDesc().implicit_defs(), P: isVCC);
362
363 if (IsSetPC) {
364 // All SGPR writes before a call/return must be flushed as the
365 // callee/caller will not will not see the hazard chain.
366 if (State.VCCHazard & HazardState::VALU)
367 Wait |= WA_VCC;
368 if (State.SALUHazards.any() || (State.VCCHazard & HazardState::SALU))
369 Wait |= WA_SALU;
370 if (State.VALUHazards.any())
371 Wait |= WA_VALU;
372 if (CullSGPRHazardsOnFunctionBoundary && State.Tracked.any()) {
373 State.Tracked.reset();
374 if (Emit)
375 insertHazardCull(MBB, MI);
376 }
377 } else {
378 // Process uses to determine required wait.
379 SeenRegs.clear();
380 for (const MachineOperand &Op : MI->all_uses()) {
381 if (Op.isImplicit() &&
382 (!HasImplicitVCC || !Op.isReg() || !isVCC(Reg: Op.getReg())))
383 continue;
384 processOperand(Op, true);
385 }
386 }
387
388 // Apply wait
389 if (Wait) {
390 unsigned Mask = AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: *ST);
391 if (Wait & WA_VCC) {
392 State.VCCHazard &= ~HazardState::VALU;
393 Mask = AMDGPU::DepCtr::encodeFieldVaVcc(Encoded: Mask, VaVcc: 0);
394 }
395 if (Wait & WA_SALU) {
396 State.SALUHazards.reset();
397 State.VCCHazard &= ~HazardState::SALU;
398 Mask = AMDGPU::DepCtr::encodeFieldSaSdst(Encoded: Mask, SaSdst: 0);
399 }
400 if (Wait & WA_VALU) {
401 State.VALUHazards.reset();
402 Mask = AMDGPU::DepCtr::encodeFieldVaSdst(Encoded: Mask, VaSdst: 0);
403 }
404 if (Emit) {
405 if (!mergeConsecutiveWaitAlus(MI, Mask)) {
406 auto NewMI = BuildMI(BB&: MBB, I: MI, MIMD: MI->getDebugLoc(),
407 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
408 .addImm(Val: Mask);
409 updateGetPCBundle(NewMI);
410 }
411 Emitted = true;
412 }
413 }
414
415 // On return from a call SGPR state is unknown, so all potential hazards.
416 if (MI->isCall() && !CullSGPRHazardsOnFunctionBoundary)
417 State.Tracked.set();
418
419 // Update hazards based on defs.
420 SeenRegs.clear();
421 for (const MachineOperand &Op : MI->all_defs()) {
422 if (Op.isImplicit() &&
423 (!HasImplicitVCC || !Op.isReg() || !isVCC(Reg: Op.getReg())))
424 continue;
425 processOperand(Op, false);
426 }
427 }
428
429 BlockHazardState &BS = BlockState[&MBB];
430 bool Changed = State != BS.Out;
431 if (Emit) {
432 assert(!Changed && "Hazard state should not change on emit pass");
433 return Emitted;
434 }
435 if (Changed)
436 BS.Out = State;
437 return Changed;
438 }
439
440 bool runWaitMerging(MachineFunction &MF) {
441 // Perform per-block merging of existing s_waitcnt_depctr instructions.
442 // Track set of SGPR writes before a given wait instruction, and search
443 // for reads of these SGPRs.
444 // Move the wait to just before the read to improve pipelining.
445 // If no related reads occur before subsequent wait then merged waits.
446 const unsigned ConstantMaskBits = AMDGPU::DepCtr::encodeFieldSaSdst(
447 Encoded: AMDGPU::DepCtr::encodeFieldVaSdst(
448 Encoded: AMDGPU::DepCtr::encodeFieldVaVcc(VaVcc: 0, STI: *ST), VaSdst: 0),
449 SaSdst: 0);
450 const unsigned VccLoIdx = *sgprNumber(Reg: AMDGPU::VCC_LO, TRI: *TRI);
451 const unsigned VccHiIdx = *sgprNumber(Reg: AMDGPU::VCC_HI, TRI: *TRI);
452 bool Changed = false;
453 for (MachineBasicBlock &MBB : MF) {
454 SmallBitVector WriteSet(128), PendingSALUWriteSet(128),
455 PendingVALUWriteSet(128);
456 MachineInstr *PrevWait = nullptr;
457
458 auto CommitWrites = [&](unsigned Mask) {
459 if (!AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Mask))
460 WriteSet |= PendingSALUWriteSet;
461 bool VccLoBit = WriteSet[VccLoIdx];
462 bool VccHiBit = WriteSet[VccHiIdx];
463 if (!AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Mask)) {
464 // Apply pending VALU set minus VCC bits
465 WriteSet |= PendingVALUWriteSet;
466 WriteSet[VccLoIdx] = VccLoBit;
467 WriteSet[VccHiIdx] = VccHiBit;
468 }
469 if (!AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Mask)) {
470 WriteSet[VccLoIdx] = VccLoBit || PendingVALUWriteSet[VccLoIdx];
471 WriteSet[VccHiIdx] = VccHiBit || PendingVALUWriteSet[VccHiIdx];
472 }
473 // Clear all pending writes
474 PendingSALUWriteSet.reset();
475 PendingVALUWriteSet.reset();
476 };
477
478 for (MachineInstr &MI : MBB) {
479 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
480 (MI.getOperand(i: 0).getImm() & ConstantMaskBits) ==
481 ConstantMaskBits) {
482 if (PrevWait) {
483 // Merge previous wait into this one.
484 MachineOperand &MaskOp = MI.getOperand(i: 0);
485 MaskOp.setImm(
486 mergeMasks(Mask1: PrevWait->getOperand(i: 0).getImm(), Mask2: MaskOp.getImm()));
487 PrevWait->eraseFromParent();
488 Changed = true;
489 } else {
490 // Starting a new region using fresh write set.
491 WriteSet.reset();
492 }
493 CommitWrites(MI.getOperand(i: 0).getImm());
494 PrevWait = &MI;
495 continue;
496 }
497
498 // Do not optimize over branches or terminators
499 if (PrevWait && (MI.isCall() || MI.isReturn() || MI.isBranch() ||
500 MI.isTerminator())) {
501 PrevWait->moveBefore(MovePos: &MI);
502 PrevWait = nullptr;
503 Changed = true;
504 }
505 if (MI.isTerminator())
506 break;
507
508 // Note: test for meta instructions after terminators.
509 // Required to handle terminator meta instruction.
510 if (MI.isMetaInstruction())
511 continue;
512
513 const bool IsVALU = SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false);
514 const bool IsSALU = SIInstrInfo::isSALU(MI);
515 if (!IsVALU && !IsSALU)
516 continue;
517
518 for (const MachineOperand &Op : MI.operands()) {
519 if (!Op.isReg())
520 continue;
521 Register Reg = Op.getReg();
522 if (!TRI->isSGPRReg(MRI: *MRI, Reg))
523 continue;
524
525 std::optional<unsigned> RegNumber = sgprNumber(Reg, TRI: *TRI);
526 if (!RegNumber)
527 continue;
528 unsigned RegN = *RegNumber;
529 unsigned SGPRCount =
530 AMDGPU::getRegBitWidth(RC: *TRI->getRegClassForReg(MRI: *MRI, Reg)) / 32;
531
532 if (Op.isDef()) {
533 if (IsSALU)
534 PendingSALUWriteSet.set(I: RegN, E: RegN + SGPRCount);
535 else
536 PendingVALUWriteSet.set(I: RegN, E: RegN + SGPRCount);
537 continue;
538 }
539
540 if (PrevWait &&
541 WriteSet.find_prev(PriorTo: RegN + SGPRCount) >= (signed)RegN) {
542 // Move the wait to here, the last point it can be valid
543 PrevWait->moveBefore(MovePos: &MI);
544 PrevWait = nullptr;
545 Changed = true;
546 }
547 }
548 }
549 }
550 return Changed;
551 }
552
553 bool run(MachineFunction &MF) {
554 ST = &MF.getSubtarget<GCNSubtarget>();
555 if (!ST->hasVALUReadSGPRHazard() && !ST->hasVALUMaskWriteHazard())
556 return false;
557
558 // Parse settings
559 CullSGPRHazardsOnFunctionBoundary = GlobalCullSGPRHazardsOnFunctionBoundary;
560 CullSGPRHazardsAtMemWait = GlobalCullSGPRHazardsAtMemWait;
561 CullSGPRHazardsMemWaitThreshold = GlobalCullSGPRHazardsMemWaitThreshold;
562
563 if (!GlobalCullSGPRHazardsOnFunctionBoundary.getNumOccurrences())
564 CullSGPRHazardsOnFunctionBoundary =
565 MF.getFunction().hasFnAttribute(Kind: "amdgpu-sgpr-hazard-boundary-cull");
566 if (!GlobalCullSGPRHazardsAtMemWait.getNumOccurrences())
567 CullSGPRHazardsAtMemWait =
568 MF.getFunction().hasFnAttribute(Kind: "amdgpu-sgpr-hazard-mem-wait-cull");
569 if (!GlobalCullSGPRHazardsMemWaitThreshold.getNumOccurrences())
570 CullSGPRHazardsMemWaitThreshold =
571 MF.getFunction().getFnAttributeAsParsedInteger(
572 Kind: "amdgpu-sgpr-hazard-mem-wait-cull-threshold",
573 Default: CullSGPRHazardsMemWaitThreshold);
574
575 TII = ST->getInstrInfo();
576 TRI = ST->getRegisterInfo();
577 MRI = &MF.getRegInfo();
578 DsNopCount = ST->isWave64() ? WAVE64_NOPS : WAVE32_NOPS;
579
580 // VALU mask write hazards have already been handled, but this pass
581 // performs a forward scan to optimize them.
582 if (ST->hasVALUMaskWriteHazard())
583 return ST->isWave64() ? runWaitMerging(MF) : false;
584
585 auto CallingConv = MF.getFunction().getCallingConv();
586 if (!AMDGPU::isEntryFunctionCC(CC: CallingConv) &&
587 !CullSGPRHazardsOnFunctionBoundary) {
588 // Callee must consider all SGPRs as tracked.
589 LLVM_DEBUG(dbgs() << "Is called function, track all SGPRs.\n");
590 MachineBasicBlock &EntryBlock = MF.front();
591 BlockState[&EntryBlock].In.Tracked.set();
592 }
593
594 // Calculate the hazard state for each basic block.
595 // Iterate until a fixed point is reached.
596 // Fixed point is guaranteed as merge function only ever increases
597 // the hazard set, and all backedges will cause a merge.
598 //
599 // Note: we have to take care of the entry block as this technically
600 // has an edge from outside the function. Failure to treat this as
601 // a merge could prevent fixed point being reached.
602 SetVector<MachineBasicBlock *> Worklist;
603 for (auto &MBB : reverse(C&: MF))
604 Worklist.insert(X: &MBB);
605 while (!Worklist.empty()) {
606 auto &MBB = *Worklist.pop_back_val();
607 bool Changed = runOnMachineBasicBlock(MBB, Emit: false);
608 if (Changed) {
609 // Note: take a copy of state here in case it is reallocated by map
610 HazardState NewState = BlockState[&MBB].Out;
611 // Propagate to all successor blocks
612 for (auto Succ : MBB.successors()) {
613 // We only need to merge hazards at CFG merge points.
614 auto &SuccState = BlockState[Succ];
615 if (Succ->getSinglePredecessor() && !Succ->isEntryBlock()) {
616 if (SuccState.In != NewState) {
617 SuccState.In = NewState;
618 Worklist.insert(X: Succ);
619 }
620 } else if (SuccState.In.merge(RHS: NewState)) {
621 Worklist.insert(X: Succ);
622 }
623 }
624 }
625 }
626
627 LLVM_DEBUG(dbgs() << "Emit s_wait_alu instructions\n");
628
629 // Final to emit wait instructions.
630 bool Changed = false;
631 for (auto &MBB : MF)
632 Changed |= runOnMachineBasicBlock(MBB, Emit: true);
633
634 BlockState.clear();
635 return Changed;
636 }
637};
638
639class AMDGPUWaitSGPRHazardsLegacy : public MachineFunctionPass {
640public:
641 static char ID;
642
643 AMDGPUWaitSGPRHazardsLegacy() : MachineFunctionPass(ID) {}
644
645 bool runOnMachineFunction(MachineFunction &MF) override {
646 return AMDGPUWaitSGPRHazards().run(MF);
647 }
648
649 void getAnalysisUsage(AnalysisUsage &AU) const override {
650 AU.setPreservesCFG();
651 MachineFunctionPass::getAnalysisUsage(AU);
652 }
653};
654
655} // namespace
656
657char AMDGPUWaitSGPRHazardsLegacy::ID = 0;
658
659char &llvm::AMDGPUWaitSGPRHazardsLegacyID = AMDGPUWaitSGPRHazardsLegacy::ID;
660
661INITIALIZE_PASS(AMDGPUWaitSGPRHazardsLegacy, DEBUG_TYPE,
662 "AMDGPU Insert waits for SGPR read hazards", false, false)
663
664PreservedAnalyses
665AMDGPUWaitSGPRHazardsPass::run(MachineFunction &MF,
666 MachineFunctionAnalysisManager &MFAM) {
667 if (AMDGPUWaitSGPRHazards().run(MF))
668 return getMachineFunctionPassPreservedAnalyses();
669 return PreservedAnalyses::all();
670}
671