1//===- AMDGPUWaitSGPRHazards.cpp - Insert waits for SGPR read hazards -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Insert s_wait_alu instructions to mitigate SGPR read hazards on GFX12.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUWaitSGPRHazards.h"
15#include "AMDGPU.h"
16#include "GCNSubtarget.h"
17#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
18#include "SIInstrInfo.h"
19#include "llvm/ADT/SetVector.h"
20#include "llvm/ADT/SmallBitVector.h"
21#include "llvm/TargetParser/AMDGPUTargetParser.h"
22
23using namespace llvm;
24
25#define DEBUG_TYPE "amdgpu-wait-sgpr-hazards"
26
27static cl::opt<bool> GlobalCullSGPRHazardsOnFunctionBoundary(
28 "amdgpu-sgpr-hazard-boundary-cull", cl::init(Val: false), cl::Hidden,
29 cl::desc("Cull hazards on function boundaries"));
30
31static cl::opt<bool>
32 GlobalCullSGPRHazardsAtMemWait("amdgpu-sgpr-hazard-mem-wait-cull",
33 cl::init(Val: false), cl::Hidden,
34 cl::desc("Cull hazards on memory waits"));
35
36static cl::opt<unsigned> GlobalCullSGPRHazardsMemWaitThreshold(
37 "amdgpu-sgpr-hazard-mem-wait-cull-threshold", cl::init(Val: 8), cl::Hidden,
38 cl::desc("Number of tracked SGPRs before initiating hazard cull on memory "
39 "wait"));
40
41namespace {
42
43class AMDGPUWaitSGPRHazards {
44public:
45 const GCNSubtarget *ST;
46 const SIInstrInfo *TII;
47 const SIRegisterInfo *TRI;
48 const MachineRegisterInfo *MRI;
49 unsigned DsNopCount;
50
51 bool CullSGPRHazardsOnFunctionBoundary;
52 bool CullSGPRHazardsAtMemWait;
53 unsigned CullSGPRHazardsMemWaitThreshold;
54
55 AMDGPUWaitSGPRHazards() = default;
56
57 // Return the numeric ID 0-127 for a given SGPR.
58 static std::optional<unsigned> sgprNumber(Register Reg,
59 const SIRegisterInfo &TRI) {
60 switch (Reg) {
61 case AMDGPU::M0:
62 case AMDGPU::EXEC:
63 case AMDGPU::EXEC_LO:
64 case AMDGPU::EXEC_HI:
65 case AMDGPU::SGPR_NULL:
66 case AMDGPU::SGPR_NULL64:
67 case AMDGPU::SCC:
68 return {};
69 default:
70 break;
71 }
72 unsigned RegN = TRI.getHWRegIndex(Reg);
73 if (RegN > 127)
74 return {};
75 return RegN;
76 }
77
78 static inline bool isVCC(Register Reg) {
79 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI;
80 }
81
82 // Adjust global offsets for instructions bundled with S_GETPC_B64 after
83 // insertion of a new instruction.
84 static void updateGetPCBundle(MachineInstr *NewMI) {
85 if (!NewMI->isBundled())
86 return;
87
88 // Find start of bundle.
89 auto I = NewMI->getIterator();
90 while (I->isBundledWithPred())
91 I--;
92 if (I->isBundle())
93 I++;
94
95 // Bail if this is not an S_GETPC bundle.
96 if (I->getOpcode() != AMDGPU::S_GETPC_B64)
97 return;
98
99 // Update offsets of any references in the bundle.
100 const unsigned NewBytes = 4;
101 assert(NewMI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
102 "Unexpected instruction insertion in bundle");
103 auto NextMI = std::next(x: NewMI->getIterator());
104 auto End = NewMI->getParent()->end();
105 while (NextMI != End && NextMI->isBundledWithPred()) {
106 for (auto &Operand : NextMI->operands()) {
107 if (Operand.isGlobal())
108 Operand.setOffset(Operand.getOffset() + NewBytes);
109 }
110 NextMI++;
111 }
112 }
113
114 struct HazardState {
115 static constexpr unsigned None = 0;
116 static constexpr unsigned SALU = (1 << 0);
117 static constexpr unsigned VALU = (1 << 1);
118
119 std::bitset<64> Tracked; // SGPR banks ever read by VALU
120 std::bitset<128> SALUHazards; // SGPRs with uncommitted values from SALU
121 std::bitset<128> VALUHazards; // SGPRs with uncommitted values from VALU
122 unsigned VCCHazard = None; // Source of current VCC writes
123 bool ActiveFlat = false; // Has unwaited flat instructions
124
125 bool merge(const HazardState &RHS) {
126 HazardState Orig(*this);
127 *this |= RHS;
128 return (*this != Orig);
129 }
130
131 bool operator==(const HazardState &RHS) const {
132 return Tracked == RHS.Tracked && SALUHazards == RHS.SALUHazards &&
133 VALUHazards == RHS.VALUHazards && VCCHazard == RHS.VCCHazard &&
134 ActiveFlat == RHS.ActiveFlat;
135 }
136
137 bool operator!=(const HazardState &RHS) const { return !(*this == RHS); }
138
139 void operator|=(const HazardState &RHS) {
140 Tracked |= RHS.Tracked;
141 SALUHazards |= RHS.SALUHazards;
142 VALUHazards |= RHS.VALUHazards;
143 VCCHazard |= RHS.VCCHazard;
144 ActiveFlat |= RHS.ActiveFlat;
145 }
146 };
147
148 struct BlockHazardState {
149 HazardState In;
150 HazardState Out;
151 };
152
153 DenseMap<const MachineBasicBlock *, BlockHazardState> BlockState;
154
155 static constexpr unsigned WAVE32_NOPS = 4;
156 static constexpr unsigned WAVE64_NOPS = 8;
157
158 void insertHazardCull(MachineBasicBlock &MBB,
159 MachineBasicBlock::instr_iterator &MI) {
160 assert(!MI->isBundled());
161 unsigned Count = DsNopCount;
162 while (Count--)
163 BuildMI(BB&: MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::DS_NOP));
164 }
165
166 unsigned mergeMasks(unsigned Mask1, unsigned Mask2) {
167 unsigned Mask = AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: *ST);
168 Mask = AMDGPU::DepCtr::encodeFieldSaSdst(
169 Encoded: Mask, SaSdst: std::min(a: AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Mask1),
170 b: AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Mask2)));
171 Mask = AMDGPU::DepCtr::encodeFieldVaVcc(
172 Encoded: Mask, VaVcc: std::min(a: AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Mask1),
173 b: AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Mask2)));
174 Mask = AMDGPU::DepCtr::encodeFieldVmVsrc(
175 Encoded: Mask, VmVsrc: std::min(a: AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: Mask1),
176 b: AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: Mask2)));
177 Mask = AMDGPU::DepCtr::encodeFieldVaSdst(
178 Encoded: Mask, VaSdst: std::min(a: AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Mask1),
179 b: AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Mask2)));
180 Mask = AMDGPU::DepCtr::encodeFieldVaVdst(
181 Encoded: Mask, VaVdst: std::min(a: AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: Mask1),
182 b: AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: Mask2)));
183 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: ST->getCPU());
184 Mask = AMDGPU::DepCtr::encodeFieldHoldCnt(
185 Encoded: Mask,
186 HoldCnt: std::min(a: AMDGPU::DepCtr::decodeFieldHoldCnt(Encoded: Mask1, Version),
187 b: AMDGPU::DepCtr::decodeFieldHoldCnt(Encoded: Mask2, Version)),
188 Version);
189 Mask = AMDGPU::DepCtr::encodeFieldVaSsrc(
190 Encoded: Mask, VaSsrc: std::min(a: AMDGPU::DepCtr::decodeFieldVaSsrc(Encoded: Mask1),
191 b: AMDGPU::DepCtr::decodeFieldVaSsrc(Encoded: Mask2)));
192 return Mask;
193 }
194
195 bool mergeConsecutiveWaitAlus(MachineBasicBlock::instr_iterator &MI,
196 unsigned Mask) {
197 auto MBB = MI->getParent();
198 if (MI == MBB->instr_begin())
199 return false;
200
201 auto It = prev_nodbg(It: MI, Begin: MBB->instr_begin());
202 if (It->getOpcode() != AMDGPU::S_WAITCNT_DEPCTR)
203 return false;
204
205 It->getOperand(i: 0).setImm(mergeMasks(Mask1: Mask, Mask2: It->getOperand(i: 0).getImm()));
206 return true;
207 }
208
209 bool runOnMachineBasicBlock(MachineBasicBlock &MBB, bool Emit) {
210 enum { WA_VALU = 0x1, WA_SALU = 0x2, WA_VCC = 0x4 };
211
212 HazardState State = BlockState[&MBB].In;
213 SmallSet<Register, 8> SeenRegs;
214 bool Emitted = false;
215 unsigned DsNops = 0;
216
217 for (MachineBasicBlock::instr_iterator MI = MBB.instr_begin(),
218 E = MBB.instr_end();
219 MI != E; ++MI) {
220 if (MI->isMetaInstruction())
221 continue;
222
223 // Clear tracked SGPRs if sufficient DS_NOPs occur
224 if (MI->getOpcode() == AMDGPU::DS_NOP) {
225 if (++DsNops >= DsNopCount)
226 State.Tracked.reset();
227 continue;
228 }
229 DsNops = 0;
230
231 // Snoop FLAT instructions to avoid adding culls before scratch/lds loads.
232 // Culls could be disproportionate in cost to load time.
233 if (SIInstrInfo::isFLAT(MI: *MI) && !SIInstrInfo::isFLATGlobal(MI: *MI))
234 State.ActiveFlat = true;
235
236 // SMEM or VMEM clears hazards
237 // FIXME: adapt to add FLAT without VALU (so !isLDSDMA())?
238 if ((SIInstrInfo::isVMEM(MI: *MI) && !SIInstrInfo::isFLAT(MI: *MI)) ||
239 SIInstrInfo::isSMRD(MI: *MI)) {
240 State.VCCHazard = HazardState::None;
241 State.SALUHazards.reset();
242 State.VALUHazards.reset();
243 continue;
244 }
245
246 // Existing S_WAITALU can clear hazards
247 if (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR) {
248 unsigned int Mask = MI->getOperand(i: 0).getImm();
249 if (AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Mask) == 0)
250 State.VCCHazard &= ~HazardState::VALU;
251 if (AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Mask) == 0) {
252 State.SALUHazards.reset();
253 State.VCCHazard &= ~HazardState::SALU;
254 }
255 if (AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Mask) == 0)
256 State.VALUHazards.reset();
257 continue;
258 }
259
260 // Snoop counter waits to insert culls
261 if (CullSGPRHazardsAtMemWait &&
262 (MI->getOpcode() == AMDGPU::S_WAIT_LOADCNT ||
263 MI->getOpcode() == AMDGPU::S_WAIT_SAMPLECNT ||
264 MI->getOpcode() == AMDGPU::S_WAIT_BVHCNT) &&
265 (MI->getOperand(i: 0).isImm() && MI->getOperand(i: 0).getImm() == 0) &&
266 (State.Tracked.count() >= CullSGPRHazardsMemWaitThreshold)) {
267 if (MI->getOpcode() == AMDGPU::S_WAIT_LOADCNT && State.ActiveFlat) {
268 State.ActiveFlat = false;
269 } else {
270 State.Tracked.reset();
271 if (Emit)
272 insertHazardCull(MBB, MI);
273 continue;
274 }
275 }
276
277 // Process only VALUs and SALUs
278 bool IsVALU = SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/true);
279 bool IsSALU = SIInstrInfo::isSALU(MI: *MI);
280 if (!IsVALU && !IsSALU)
281 continue;
282
283 unsigned Wait = 0;
284
285 auto processOperand = [&](const MachineOperand &Op, bool IsUse) {
286 if (!Op.isReg())
287 return;
288 Register Reg = Op.getReg();
289 assert(!Op.getSubReg());
290 if (!TRI->isSGPRReg(MRI: *MRI, Reg))
291 return;
292
293 // Only visit each register once
294 if (!SeenRegs.insert(V: Reg).second)
295 return;
296
297 auto RegNumber = sgprNumber(Reg, TRI: *TRI);
298 if (!RegNumber)
299 return;
300
301 // Track SGPRs by pair -- numeric ID of an 64b SGPR pair.
302 // i.e. SGPR0 = SGPR0_SGPR1 = 0, SGPR3 = SGPR2_SGPR3 = 1, etc
303 unsigned RegN = *RegNumber;
304 unsigned PairN = (RegN >> 1) & 0x3f;
305
306 // Read/write of untracked register is safe; but must record any new
307 // reads.
308 if (!State.Tracked[PairN]) {
309 if (IsVALU && IsUse)
310 State.Tracked.set(position: PairN);
311 return;
312 }
313
314 uint8_t SGPRCount =
315 AMDGPU::getRegBitWidth(RC: *TRI->getRegClassForReg(MRI: *MRI, Reg)) / 32;
316
317 if (IsUse) {
318 // SALU reading SGPR clears VALU hazards
319 if (IsSALU) {
320 if (isVCC(Reg)) {
321 if (State.VCCHazard & HazardState::VALU)
322 State.VCCHazard = HazardState::None;
323 } else {
324 State.VALUHazards.reset();
325 }
326 }
327 // Compute required waits
328 for (uint8_t RegIdx = 0; RegIdx < SGPRCount; ++RegIdx) {
329 Wait |= State.SALUHazards[RegN + RegIdx] ? WA_SALU : 0;
330 Wait |= IsVALU && State.VALUHazards[RegN + RegIdx] ? WA_VALU : 0;
331 }
332 if (isVCC(Reg) && State.VCCHazard) {
333 // Note: it's possible for both SALU and VALU to exist if VCC
334 // was updated differently by merged predecessors.
335 if (State.VCCHazard & HazardState::SALU)
336 Wait |= WA_SALU;
337 if (State.VCCHazard & HazardState::VALU)
338 Wait |= WA_VCC;
339 }
340 } else {
341 // Update hazards
342 if (isVCC(Reg)) {
343 State.VCCHazard = IsSALU ? HazardState::SALU : HazardState::VALU;
344 } else {
345 for (uint8_t RegIdx = 0; RegIdx < SGPRCount; ++RegIdx) {
346 if (IsSALU)
347 State.SALUHazards.set(position: RegN + RegIdx);
348 else
349 State.VALUHazards.set(position: RegN + RegIdx);
350 }
351 }
352 }
353 };
354
355 const bool IsSetPC =
356 (MI->isCall() || MI->isReturn() || MI->isIndirectBranch()) &&
357 MI->getOpcode() != AMDGPU::S_ENDPGM &&
358 MI->getOpcode() != AMDGPU::S_ENDPGM_SAVED;
359
360 // Only consider implicit VCC specified by instruction descriptor.
361 const bool HasImplicitVCC =
362 llvm::any_of(Range: MI->getDesc().implicit_uses(), P: isVCC) ||
363 llvm::any_of(Range: MI->getDesc().implicit_defs(), P: isVCC);
364
365 if (IsSetPC) {
366 // All SGPR writes before a call/return must be flushed as the
367 // callee/caller will not will not see the hazard chain.
368 if (State.VCCHazard & HazardState::VALU)
369 Wait |= WA_VCC;
370 if (State.SALUHazards.any() || (State.VCCHazard & HazardState::SALU))
371 Wait |= WA_SALU;
372 if (State.VALUHazards.any())
373 Wait |= WA_VALU;
374 if (CullSGPRHazardsOnFunctionBoundary && State.Tracked.any()) {
375 State.Tracked.reset();
376 if (Emit)
377 insertHazardCull(MBB, MI);
378 }
379 } else {
380 // Process uses to determine required wait.
381 SeenRegs.clear();
382 for (const MachineOperand &Op : MI->all_uses()) {
383 if (Op.isImplicit() &&
384 (!HasImplicitVCC || !Op.isReg() || !isVCC(Reg: Op.getReg())))
385 continue;
386 processOperand(Op, true);
387 }
388 }
389
390 // Apply wait
391 if (Wait) {
392 unsigned Mask = AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: *ST);
393 if (Wait & WA_VCC) {
394 State.VCCHazard &= ~HazardState::VALU;
395 Mask = AMDGPU::DepCtr::encodeFieldVaVcc(Encoded: Mask, VaVcc: 0);
396 }
397 if (Wait & WA_SALU) {
398 State.SALUHazards.reset();
399 State.VCCHazard &= ~HazardState::SALU;
400 Mask = AMDGPU::DepCtr::encodeFieldSaSdst(Encoded: Mask, SaSdst: 0);
401 }
402 if (Wait & WA_VALU) {
403 State.VALUHazards.reset();
404 Mask = AMDGPU::DepCtr::encodeFieldVaSdst(Encoded: Mask, VaSdst: 0);
405 }
406 if (Emit) {
407 if (!mergeConsecutiveWaitAlus(MI, Mask)) {
408 auto NewMI = BuildMI(BB&: MBB, I: MI, MIMD: MI->getDebugLoc(),
409 MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
410 .addImm(Val: Mask);
411 updateGetPCBundle(NewMI);
412 }
413 Emitted = true;
414 }
415 }
416
417 // On return from a call SGPR state is unknown, so all potential hazards.
418 if (MI->isCall() && !CullSGPRHazardsOnFunctionBoundary)
419 State.Tracked.set();
420
421 // Update hazards based on defs.
422 SeenRegs.clear();
423 for (const MachineOperand &Op : MI->all_defs()) {
424 if (Op.isImplicit() &&
425 (!HasImplicitVCC || !Op.isReg() || !isVCC(Reg: Op.getReg())))
426 continue;
427 processOperand(Op, false);
428 }
429 }
430
431 BlockHazardState &BS = BlockState[&MBB];
432 bool Changed = State != BS.Out;
433 if (Emit) {
434 assert(!Changed && "Hazard state should not change on emit pass");
435 return Emitted;
436 }
437 if (Changed)
438 BS.Out = State;
439 return Changed;
440 }
441
442 bool runWaitMerging(MachineFunction &MF) {
443 // Perform per-block merging of existing s_waitcnt_depctr instructions.
444 // Track set of SGPR writes before a given wait instruction, and search
445 // for reads of these SGPRs.
446 // Move the wait to just before the read to improve pipelining.
447 // If no related reads occur before subsequent wait then merged waits.
448 const unsigned ConstantMaskBits = AMDGPU::DepCtr::encodeFieldSaSdst(
449 Encoded: AMDGPU::DepCtr::encodeFieldVaSdst(
450 Encoded: AMDGPU::DepCtr::encodeFieldVaVcc(VaVcc: 0, STI: *ST), VaSdst: 0),
451 SaSdst: 0);
452 const unsigned VccLoIdx = *sgprNumber(Reg: AMDGPU::VCC_LO, TRI: *TRI);
453 const unsigned VccHiIdx = *sgprNumber(Reg: AMDGPU::VCC_HI, TRI: *TRI);
454 bool Changed = false;
455 for (MachineBasicBlock &MBB : MF) {
456 SmallBitVector WriteSet(128), PendingSALUWriteSet(128),
457 PendingVALUWriteSet(128);
458 MachineInstr *PrevWait = nullptr;
459
460 auto CommitWrites = [&](unsigned Mask) {
461 if (!AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Mask))
462 WriteSet |= PendingSALUWriteSet;
463 bool VccLoBit = WriteSet[VccLoIdx];
464 bool VccHiBit = WriteSet[VccHiIdx];
465 if (!AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Mask)) {
466 // Apply pending VALU set minus VCC bits
467 WriteSet |= PendingVALUWriteSet;
468 WriteSet[VccLoIdx] = VccLoBit;
469 WriteSet[VccHiIdx] = VccHiBit;
470 }
471 if (!AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Mask)) {
472 WriteSet[VccLoIdx] = VccLoBit || PendingVALUWriteSet[VccLoIdx];
473 WriteSet[VccHiIdx] = VccHiBit || PendingVALUWriteSet[VccHiIdx];
474 }
475 // Clear all pending writes
476 PendingSALUWriteSet.reset();
477 PendingVALUWriteSet.reset();
478 };
479
480 for (MachineInstr &MI : MBB) {
481 if (MI.isMetaInstruction())
482 continue;
483
484 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
485 (MI.getOperand(i: 0).getImm() & ConstantMaskBits) ==
486 ConstantMaskBits) {
487 if (PrevWait) {
488 // Merge previous wait into this one.
489 MachineOperand &MaskOp = MI.getOperand(i: 0);
490 MaskOp.setImm(
491 mergeMasks(Mask1: PrevWait->getOperand(i: 0).getImm(), Mask2: MaskOp.getImm()));
492 PrevWait->eraseFromParent();
493 Changed = true;
494 } else {
495 // Starting a new region using fresh write set.
496 WriteSet.reset();
497 }
498 CommitWrites(MI.getOperand(i: 0).getImm());
499 PrevWait = &MI;
500 continue;
501 }
502
503 // Do not optimize over branches
504 if (PrevWait && (MI.isCall() || MI.isReturn() || MI.isBranch())) {
505 PrevWait->moveBefore(MovePos: &MI);
506 PrevWait = nullptr;
507 Changed = true;
508 }
509
510 const bool IsVALU = SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false);
511 const bool IsSALU = SIInstrInfo::isSALU(MI);
512 if (!IsVALU && !IsSALU)
513 continue;
514
515 for (const MachineOperand &Op : MI.operands()) {
516 if (!Op.isReg())
517 continue;
518 Register Reg = Op.getReg();
519 if (!TRI->isSGPRReg(MRI: *MRI, Reg))
520 continue;
521
522 std::optional<unsigned> RegNumber = sgprNumber(Reg, TRI: *TRI);
523 if (!RegNumber)
524 continue;
525 unsigned RegN = *RegNumber;
526 unsigned SGPRCount =
527 AMDGPU::getRegBitWidth(RC: *TRI->getRegClassForReg(MRI: *MRI, Reg)) / 32;
528
529 if (Op.isDef()) {
530 if (IsSALU)
531 PendingSALUWriteSet.set(I: RegN, E: RegN + SGPRCount);
532 else
533 PendingVALUWriteSet.set(I: RegN, E: RegN + SGPRCount);
534 continue;
535 }
536
537 if (PrevWait &&
538 WriteSet.find_prev(PriorTo: RegN + SGPRCount) >= (signed)RegN) {
539 // Move the wait to here, the last point it can be valid
540 PrevWait->moveBefore(MovePos: &MI);
541 PrevWait = nullptr;
542 Changed = true;
543 }
544 }
545 }
546 }
547 return Changed;
548 }
549
550 bool run(MachineFunction &MF) {
551 ST = &MF.getSubtarget<GCNSubtarget>();
552 if (!ST->hasVALUReadSGPRHazard() && !ST->hasVALUMaskWriteHazard())
553 return false;
554
555 // Parse settings
556 CullSGPRHazardsOnFunctionBoundary = GlobalCullSGPRHazardsOnFunctionBoundary;
557 CullSGPRHazardsAtMemWait = GlobalCullSGPRHazardsAtMemWait;
558 CullSGPRHazardsMemWaitThreshold = GlobalCullSGPRHazardsMemWaitThreshold;
559
560 if (!GlobalCullSGPRHazardsOnFunctionBoundary.getNumOccurrences())
561 CullSGPRHazardsOnFunctionBoundary =
562 MF.getFunction().hasFnAttribute(Kind: "amdgpu-sgpr-hazard-boundary-cull");
563 if (!GlobalCullSGPRHazardsAtMemWait.getNumOccurrences())
564 CullSGPRHazardsAtMemWait =
565 MF.getFunction().hasFnAttribute(Kind: "amdgpu-sgpr-hazard-mem-wait-cull");
566 if (!GlobalCullSGPRHazardsMemWaitThreshold.getNumOccurrences())
567 CullSGPRHazardsMemWaitThreshold =
568 MF.getFunction().getFnAttributeAsParsedInteger(
569 Kind: "amdgpu-sgpr-hazard-mem-wait-cull-threshold",
570 Default: CullSGPRHazardsMemWaitThreshold);
571
572 TII = ST->getInstrInfo();
573 TRI = ST->getRegisterInfo();
574 MRI = &MF.getRegInfo();
575 DsNopCount = ST->isWave64() ? WAVE64_NOPS : WAVE32_NOPS;
576
577 // VALU mask write hazards have already been handled, but this pass
578 // performs a forward scan to optimize them.
579 if (ST->hasVALUMaskWriteHazard())
580 return ST->isWave64() ? runWaitMerging(MF) : false;
581
582 auto CallingConv = MF.getFunction().getCallingConv();
583 if (!AMDGPU::isEntryFunctionCC(CC: CallingConv) &&
584 !CullSGPRHazardsOnFunctionBoundary) {
585 // Callee must consider all SGPRs as tracked.
586 LLVM_DEBUG(dbgs() << "Is called function, track all SGPRs.\n");
587 MachineBasicBlock &EntryBlock = MF.front();
588 BlockState[&EntryBlock].In.Tracked.set();
589 }
590
591 // Calculate the hazard state for each basic block.
592 // Iterate until a fixed point is reached.
593 // Fixed point is guaranteed as merge function only ever increases
594 // the hazard set, and all backedges will cause a merge.
595 //
596 // Note: we have to take care of the entry block as this technically
597 // has an edge from outside the function. Failure to treat this as
598 // a merge could prevent fixed point being reached.
599 SetVector<MachineBasicBlock *> Worklist;
600 for (auto &MBB : reverse(C&: MF))
601 Worklist.insert(X: &MBB);
602 while (!Worklist.empty()) {
603 auto &MBB = *Worklist.pop_back_val();
604 bool Changed = runOnMachineBasicBlock(MBB, Emit: false);
605 if (Changed) {
606 // Note: take a copy of state here in case it is reallocated by map
607 HazardState NewState = BlockState[&MBB].Out;
608 // Propagate to all successor blocks
609 for (auto Succ : MBB.successors()) {
610 // We only need to merge hazards at CFG merge points.
611 auto &SuccState = BlockState[Succ];
612 if (Succ->getSinglePredecessor() && !Succ->isEntryBlock()) {
613 if (SuccState.In != NewState) {
614 SuccState.In = NewState;
615 Worklist.insert(X: Succ);
616 }
617 } else if (SuccState.In.merge(RHS: NewState)) {
618 Worklist.insert(X: Succ);
619 }
620 }
621 }
622 }
623
624 LLVM_DEBUG(dbgs() << "Emit s_wait_alu instructions\n");
625
626 // Final to emit wait instructions.
627 bool Changed = false;
628 for (auto &MBB : MF)
629 Changed |= runOnMachineBasicBlock(MBB, Emit: true);
630
631 BlockState.clear();
632 return Changed;
633 }
634};
635
636class AMDGPUWaitSGPRHazardsLegacy : public MachineFunctionPass {
637public:
638 static char ID;
639
640 AMDGPUWaitSGPRHazardsLegacy() : MachineFunctionPass(ID) {}
641
642 bool runOnMachineFunction(MachineFunction &MF) override {
643 return AMDGPUWaitSGPRHazards().run(MF);
644 }
645
646 void getAnalysisUsage(AnalysisUsage &AU) const override {
647 AU.setPreservesCFG();
648 MachineFunctionPass::getAnalysisUsage(AU);
649 }
650};
651
652} // namespace
653
654char AMDGPUWaitSGPRHazardsLegacy::ID = 0;
655
656char &llvm::AMDGPUWaitSGPRHazardsLegacyID = AMDGPUWaitSGPRHazardsLegacy::ID;
657
658INITIALIZE_PASS(AMDGPUWaitSGPRHazardsLegacy, DEBUG_TYPE,
659 "AMDGPU Insert waits for SGPR read hazards", false, false)
660
661PreservedAnalyses
662AMDGPUWaitSGPRHazardsPass::run(MachineFunction &MF,
663 MachineFunctionAnalysisManager &MFAM) {
664 if (AMDGPUWaitSGPRHazards().run(MF))
665 return getMachineFunctionPassPreservedAnalyses();
666 return PreservedAnalyses::all();
667}
668